Fixes ensuring VBMCs are actually running

We currently start VBMCs using CLI due to issues with the VBMC python
lib.  However when we start them, there is no check to make sure they
are actually changed to 'running' status.  This patch adds logic to
check (up to 5 times) to ensure each VBMC is running or raises an error.
Note this is for virtual deployments only.

JIRA: APEX-527

Change-Id: Iab7ee3b76292d6fc547f18c83f23c04205e9bb2e
Signed-off-by: Tim Rozet <trozet@redhat.com>

3 files changed, 64 insertions, 4 deletions

diff --git a/apex/tests/test_apex_virtual_utils.py b/apex/tests/test_apex_virtual_utils.py
index 643069f3..a9eb78dd 100644
--- a/apex/tests/test_apex_virtual_utils.py
+++ b/apex/tests/test_apex_virtual_utils.py
@@ -12,6 +12,7 @@ import unittest
 
 from mock import patch
 
+from apex.virtual.exceptions import ApexVirtualException
 from apex.virtual.utils import DEFAULT_VIRT_IP
 from apex.virtual.utils import get_virt_ip
 from apex.virtual.utils import generate_inventory
@@ -66,13 +67,30 @@ class TestVirtualUtils(unittest.TestCase):
         assert_is_instance(generate_inventory('target_file', ha_enabled=True),
                            dict)
 
+    @patch('apex.virtual.utils.get_virt_ip')
+    @patch('apex.virtual.utils.subprocess.check_output')
     @patch('apex.virtual.utils.iptc')
     @patch('apex.virtual.utils.subprocess.check_call')
     @patch('apex.virtual.utils.vbmc_lib')
-    def test_host_setup(self, mock_vbmc_lib, mock_subprocess, mock_iptc):
+    def test_host_setup(self, mock_vbmc_lib, mock_subprocess, mock_iptc,
+                        mock_check_output, mock_get_virt_ip):
+        mock_get_virt_ip.return_value = '192.168.122.1'
+        mock_check_output.return_value = b'blah |dummy \nstatus | running'
         host_setup({'test': 2468})
         mock_subprocess.assert_called_with(['vbmc', 'start', 'test'])
 
+    @patch('apex.virtual.utils.get_virt_ip')
+    @patch('apex.virtual.utils.subprocess.check_output')
+    @patch('apex.virtual.utils.iptc')
+    @patch('apex.virtual.utils.subprocess.check_call')
+    @patch('apex.virtual.utils.vbmc_lib')
+    def test_host_setup_vbmc_fails(self, mock_vbmc_lib, mock_subprocess,
+                                   mock_iptc, mock_check_output,
+                                   mock_get_virt_ip):
+        mock_get_virt_ip.return_value = '192.168.122.1'
+        mock_check_output.return_value = b'blah |dummy \nstatus | stopped'
+        assert_raises(ApexVirtualException, host_setup, {'test': 2468})
+
     @patch('apex.virtual.utils.iptc')
     @patch('apex.virtual.utils.subprocess.check_call')
     @patch('apex.virtual.utils.vbmc_lib')
diff --git a/apex/virtual/exceptions.py b/apex/virtual/exceptions.py
new file mode 100644
index 00000000..e3dff51a
--- /dev/null
+++ b/apex/virtual/exceptions.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2018 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class ApexVirtualException(Exception):
+    pass
diff --git a/apex/virtual/utils.py b/apex/virtual/utils.py
index 226af1b5..8b24bc40 100644
--- a/apex/virtual/utils.py
+++ b/apex/virtual/utils.py
@@ -18,6 +18,8 @@ import xml.etree.ElementTree as ET
 
 from apex.common import utils as common_utils
 from apex.virtual import configure_vm as vm_lib
+from apex.virtual import exceptions as exc
+from time import sleep
 from virtualbmc import manager as vbmc_lib
 
 DEFAULT_RAM = 8192
@@ -131,11 +133,39 @@ def host_setup(node):
         chain.insert_rule(rule)
         try:
             subprocess.check_call(['vbmc', 'start', name])
-            logging.debug("Started vbmc for domain {}".format(name))
+            logging.debug("Started VBMC for domain {}".format(name))
         except subprocess.CalledProcessError:
-            logging.error("Failed to start vbmc for {}".format(name))
+            logging.error("Failed to start VBMC for {}".format(name))
             raise
-    logging.debug('vmbcs setup: {}'.format(vbmc_manager.list()))
+
+        logging.info("Checking VBMC {} is up".format(name))
+        is_running = False
+        for x in range(0, 4):
+            logging.debug("Polling to see if VBMC is up, attempt {}".format(x))
+            try:
+                output = subprocess.check_output(['vbmc', 'show', name],
+                                                 stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError:
+                logging.warning('Unable to issue "vbmc show" cmd')
+                continue
+            for line in output.decode('utf-8').split('\n'):
+                if 'status' in line:
+                    if 'running' in line:
+                        is_running = True
+                        break
+                    else:
+                        logging.debug('VBMC status is not "running"')
+                    break
+            if is_running:
+                break
+            sleep(1)
+        if is_running:
+            logging.info("VBMC {} is up and running".format(name))
+        else:
+            logging.error("Failed to verify VBMC is running")
+            raise exc.ApexVirtualException("Failed to bring up vbmc "
+                                           "{}".format(name))
+    logging.debug('VBMCs setup: {}'.format(vbmc_manager.list()))
 
 
 def virt_customize(ops, target):