summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTim Rozet <trozet@redhat.com>2018-02-16 14:16:59 -0500
committerTim Rozet <trozet@redhat.com>2018-02-16 14:22:46 -0500
commit0e2c8a5a3dc5919df7d906be951331372775eeff (patch)
treeb137935b91975d336e2d5fdabb7c2c973a6d4417
parentd75cbb74191e32e3ff996604e30a2954d889687b (diff)
Fixes ensuring VBMCs are actually running
We currently start VBMCs using CLI due to issues with the VBMC python lib. However when we start them, there is no check to make sure they are actually changed to 'running' status. This patch adds logic to check (up to 5 times) to ensure each VBMC is running or raises an error. Note this is for virtual deployments only. JIRA: APEX-527 Change-Id: Iab7ee3b76292d6fc547f18c83f23c04205e9bb2e Signed-off-by: Tim Rozet <trozet@redhat.com>
-rw-r--r--apex/tests/test_apex_virtual_utils.py20
-rw-r--r--apex/virtual/exceptions.py12
-rw-r--r--apex/virtual/utils.py36
3 files changed, 64 insertions, 4 deletions
diff --git a/apex/tests/test_apex_virtual_utils.py b/apex/tests/test_apex_virtual_utils.py
index 643069f3..a9eb78dd 100644
--- a/apex/tests/test_apex_virtual_utils.py
+++ b/apex/tests/test_apex_virtual_utils.py
@@ -12,6 +12,7 @@ import unittest
from mock import patch
+from apex.virtual.exceptions import ApexVirtualException
from apex.virtual.utils import DEFAULT_VIRT_IP
from apex.virtual.utils import get_virt_ip
from apex.virtual.utils import generate_inventory
@@ -66,13 +67,30 @@ class TestVirtualUtils(unittest.TestCase):
assert_is_instance(generate_inventory('target_file', ha_enabled=True),
dict)
+ @patch('apex.virtual.utils.get_virt_ip')
+ @patch('apex.virtual.utils.subprocess.check_output')
@patch('apex.virtual.utils.iptc')
@patch('apex.virtual.utils.subprocess.check_call')
@patch('apex.virtual.utils.vbmc_lib')
- def test_host_setup(self, mock_vbmc_lib, mock_subprocess, mock_iptc):
+ def test_host_setup(self, mock_vbmc_lib, mock_subprocess, mock_iptc,
+ mock_check_output, mock_get_virt_ip):
+ mock_get_virt_ip.return_value = '192.168.122.1'
+ mock_check_output.return_value = b'blah |dummy \nstatus | running'
host_setup({'test': 2468})
mock_subprocess.assert_called_with(['vbmc', 'start', 'test'])
+ @patch('apex.virtual.utils.get_virt_ip')
+ @patch('apex.virtual.utils.subprocess.check_output')
+ @patch('apex.virtual.utils.iptc')
+ @patch('apex.virtual.utils.subprocess.check_call')
+ @patch('apex.virtual.utils.vbmc_lib')
+ def test_host_setup_vbmc_fails(self, mock_vbmc_lib, mock_subprocess,
+ mock_iptc, mock_check_output,
+ mock_get_virt_ip):
+ mock_get_virt_ip.return_value = '192.168.122.1'
+ mock_check_output.return_value = b'blah |dummy \nstatus | stopped'
+ assert_raises(ApexVirtualException, host_setup, {'test': 2468})
+
@patch('apex.virtual.utils.iptc')
@patch('apex.virtual.utils.subprocess.check_call')
@patch('apex.virtual.utils.vbmc_lib')
diff --git a/apex/virtual/exceptions.py b/apex/virtual/exceptions.py
new file mode 100644
index 00000000..e3dff51a
--- /dev/null
+++ b/apex/virtual/exceptions.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2018 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class ApexVirtualException(Exception):
+ pass
diff --git a/apex/virtual/utils.py b/apex/virtual/utils.py
index 226af1b5..8b24bc40 100644
--- a/apex/virtual/utils.py
+++ b/apex/virtual/utils.py
@@ -18,6 +18,8 @@ import xml.etree.ElementTree as ET
from apex.common import utils as common_utils
from apex.virtual import configure_vm as vm_lib
+from apex.virtual import exceptions as exc
+from time import sleep
from virtualbmc import manager as vbmc_lib
DEFAULT_RAM = 8192
@@ -131,11 +133,39 @@ def host_setup(node):
chain.insert_rule(rule)
try:
subprocess.check_call(['vbmc', 'start', name])
- logging.debug("Started vbmc for domain {}".format(name))
+ logging.debug("Started VBMC for domain {}".format(name))
except subprocess.CalledProcessError:
- logging.error("Failed to start vbmc for {}".format(name))
+ logging.error("Failed to start VBMC for {}".format(name))
raise
- logging.debug('vmbcs setup: {}'.format(vbmc_manager.list()))
+
+ logging.info("Checking VBMC {} is up".format(name))
+ is_running = False
+ for x in range(0, 4):
+ logging.debug("Polling to see if VBMC is up, attempt {}".format(x))
+ try:
+ output = subprocess.check_output(['vbmc', 'show', name],
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError:
+ logging.warning('Unable to issue "vbmc show" cmd')
+ continue
+ for line in output.decode('utf-8').split('\n'):
+ if 'status' in line:
+ if 'running' in line:
+ is_running = True
+ break
+ else:
+ logging.debug('VBMC status is not "running"')
+ break
+ if is_running:
+ break
+ sleep(1)
+ if is_running:
+ logging.info("VBMC {} is up and running".format(name))
+ else:
+ logging.error("Failed to verify VBMC is running")
+ raise exc.ApexVirtualException("Failed to bring up vbmc "
+ "{}".format(name))
+ logging.debug('VBMCs setup: {}'.format(vbmc_manager.list()))
def virt_customize(ops, target):