Merge "Fixes ensuring VBMCs are actually running"

3 files changed, 64 insertions, 4 deletions

diff --git a/apex/tests/test_apex_virtual_utils.py b/apex/tests/test_apex_virtual_utils.py
index 643069f3..a9eb78dd 100644
--- a/apex/tests/test_apex_virtual_utils.py
+++ b/apex/tests/test_apex_virtual_utils.py
@@ -12,6 +12,7 @@ import unittest
 
 from mock import patch
 
+from apex.virtual.exceptions import ApexVirtualException
 from apex.virtual.utils import DEFAULT_VIRT_IP
 from apex.virtual.utils import get_virt_ip
 from apex.virtual.utils import generate_inventory
@@ -66,13 +67,30 @@ class TestVirtualUtils(unittest.TestCase):
         assert_is_instance(generate_inventory('target_file', ha_enabled=True),
                            dict)
 
+    @patch('apex.virtual.utils.get_virt_ip')
+    @patch('apex.virtual.utils.subprocess.check_output')
     @patch('apex.virtual.utils.iptc')
     @patch('apex.virtual.utils.subprocess.check_call')
     @patch('apex.virtual.utils.vbmc_lib')
-    def test_host_setup(self, mock_vbmc_lib, mock_subprocess, mock_iptc):
+    def test_host_setup(self, mock_vbmc_lib, mock_subprocess, mock_iptc,
+                        mock_check_output, mock_get_virt_ip):
+        mock_get_virt_ip.return_value = '192.168.122.1'
+        mock_check_output.return_value = b'blah |dummy \nstatus | running'
         host_setup({'test': 2468})
         mock_subprocess.assert_called_with(['vbmc', 'start', 'test'])
 
+    @patch('apex.virtual.utils.get_virt_ip')
+    @patch('apex.virtual.utils.subprocess.check_output')
+    @patch('apex.virtual.utils.iptc')
+    @patch('apex.virtual.utils.subprocess.check_call')
+    @patch('apex.virtual.utils.vbmc_lib')
+    def test_host_setup_vbmc_fails(self, mock_vbmc_lib, mock_subprocess,
+                                   mock_iptc, mock_check_output,
+                                   mock_get_virt_ip):
+        mock_get_virt_ip.return_value = '192.168.122.1'
+        mock_check_output.return_value = b'blah |dummy \nstatus | stopped'
+        assert_raises(ApexVirtualException, host_setup, {'test': 2468})
+
     @patch('apex.virtual.utils.iptc')
     @patch('apex.virtual.utils.subprocess.check_call')
     @patch('apex.virtual.utils.vbmc_lib')
diff --git a/apex/virtual/exceptions.py b/apex/virtual/exceptions.py
new file mode 100644
index 00000000..e3dff51a
--- /dev/null
+++ b/apex/virtual/exceptions.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2018 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class ApexVirtualException(Exception):
+    pass
diff --git a/apex/virtual/utils.py b/apex/virtual/utils.py
index 226af1b5..8b24bc40 100644
--- a/apex/virtual/utils.py
+++ b/apex/virtual/utils.py
@@ -18,6 +18,8 @@ import xml.etree.ElementTree as ET
 
 from apex.common import utils as common_utils
 from apex.virtual import configure_vm as vm_lib
+from apex.virtual import exceptions as exc
+from time import sleep
 from virtualbmc import manager as vbmc_lib
 
 DEFAULT_RAM = 8192
@@ -131,11 +133,39 @@ def host_setup(node):
         chain.insert_rule(rule)
         try:
             subprocess.check_call(['vbmc', 'start', name])
-            logging.debug("Started vbmc for domain {}".format(name))
+            logging.debug("Started VBMC for domain {}".format(name))
         except subprocess.CalledProcessError:
-            logging.error("Failed to start vbmc for {}".format(name))
+            logging.error("Failed to start VBMC for {}".format(name))
             raise
-    logging.debug('vmbcs setup: {}'.format(vbmc_manager.list()))
+
+        logging.info("Checking VBMC {} is up".format(name))
+        is_running = False
+        for x in range(0, 4):
+            logging.debug("Polling to see if VBMC is up, attempt {}".format(x))
+            try:
+                output = subprocess.check_output(['vbmc', 'show', name],
+                                                 stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError:
+                logging.warning('Unable to issue "vbmc show" cmd')
+                continue
+            for line in output.decode('utf-8').split('\n'):
+                if 'status' in line:
+                    if 'running' in line:
+                        is_running = True
+                        break
+                    else:
+                        logging.debug('VBMC status is not "running"')
+                    break
+            if is_running:
+                break
+            sleep(1)
+        if is_running:
+            logging.info("VBMC {} is up and running".format(name))
+        else:
+            logging.error("Failed to verify VBMC is running")
+            raise exc.ApexVirtualException("Failed to bring up vbmc "
+                                           "{}".format(name))
+    logging.debug('VBMCs setup: {}'.format(vbmc_manager.list()))
 
 
 def virt_customize(ops, target):