refactor failure inject

JIRA: DOCTOR-116

Change-Id: I14deda4ccb47414cff139a522a9196b68e92500e
Signed-off-by: dongwenjuan <dong.wenjuan@zte.com.cn>

1 files changed, 70 insertions, 8 deletions

diff --git a/tests/main.py b/tests/main.py
index db2fafd9..7e7c3bc2 100644
--- a/tests/main.py
+++ b/tests/main.py
@@ -8,19 +8,27 @@
 ##############################################################################
 import os
 from os.path import isfile, join
+import random
 import sys
+import time
 
 from alarm import Alarm
+from common.constants import Host
 import config
 from consumer import get_consumer
+from identity_auth import get_identity_auth
+from identity_auth import get_session
 from image import Image
 from instance import Instance
 from inspector import get_inspector
 from installer import get_installer
 import logger as doctor_log
-from user import User
 from network import Network
 from monitor import get_monitor
+from os_clients import nova_client
+from scenario.common import calculate_notification_time
+from scenario.network_failure import NetworkFault
+from user import User
 
 
 LOG = doctor_log.Logger('doctor').getLogger()
@@ -35,12 +43,17 @@ class DoctorTest(object):
         self.network = Network(self.conf, LOG)
         self.instance = Instance(self.conf, LOG)
         self.alarm = Alarm(self.conf, LOG)
+        self.installer = get_installer(self.conf, LOG)
         self.inspector = get_inspector(self.conf, LOG)
         self.monitor = get_monitor(self.conf,
                                    self.inspector.get_inspector_url(),
                                    LOG)
         self.consumer = get_consumer(self.conf, LOG)
-        self.installer = get_installer(self.conf, LOG)
+        self.fault = NetworkFault(self.conf, self.installer, LOG)
+        auth = get_identity_auth(project=self.conf.doctor_project)
+        self.nova = nova_client(self.conf.nova_version,
+                                get_session(auth=auth))
+        self.down_host = None
 
     def setup(self):
         # prepare the cloud env
@@ -63,7 +76,10 @@ class DoctorTest(object):
 
         # starting doctor sample components...
         self.inspector.start()
-        self.monitor.start()
+
+        self.down_host = self.get_host_info_for_random_vm()
+        self.monitor.start(self.down_host)
+
         self.consumer.start()
 
     def run(self):
@@ -71,30 +87,76 @@ class DoctorTest(object):
         try:
             LOG.info('doctor test starting.......')
 
+            # prepare test env
             self.setup()
 
+            # wait for aodh alarms are updated in caches for event evaluator,
+            # sleep time should be larger than event_alarm_cache_ttl(default 60)
+            time.sleep(60)
+
             # injecting host failure...
             # NOTE (umar) add INTERFACE_NAME logic to host injection
 
+            self.fault.start(self.down_host)
+            time.sleep(10)
+
             # verify the test results
             # NOTE (umar) copy remote monitor.log file when monitor=collectd
-
+            self.check_host_status(self.down_host.name, 'down')
+
+            notification_time = calculate_notification_time()
+            if notification_time < 1 and notification_time > 0:
+                LOG.info('doctor test successfully, notification_time=%s' % notification_time)
+            else:
+                LOG.error('doctor test failed, notification_time=%s' % notification_time)
+                sys.exit(1)
         except Exception as e:
             LOG.error('doctor test failed, Exception=%s' % e)
             sys.exit(1)
         finally:
             self.cleanup()
 
+    def get_host_info_for_random_vm(self):
+        num = random.randint(0, self.conf.instance_count - 1)
+        vm_name = "%s%d" % (self.conf.instance_basename, num)
+
+        servers = \
+            {getattr(server, 'name'): server
+             for server in self.nova.servers.list()}
+        server = servers.get(vm_name)
+        if not server:
+            raise \
+                Exception('Can not find instance: vm_name(%s)' % vm_name)
+        host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname')
+        host_ip = self.installer.get_host_ip_from_hostname(host_name)
+
+        LOG.info('Get host info(name:%s, ip:%s) which vm(%s) launched at'
+                 % (host_name, host_ip, vm_name))
+        return Host(host_name, host_ip)
+
+    def check_host_status(self, hostname, state):
+        service = self.nova.services.list(host=hostname, binary='nova-compute')
+        host_state = service[0].__dict__.get('state')
+        assert host_state == state
+
+    def unset_forced_down_hosts(self):
+        if self.down_host:
+            self.nova.services.force_down(self.down_host.name, 'nova-compute', False)
+            time.sleep(2)
+            self.check_host_status(self.down_host.name, 'up')
+
     def cleanup(self):
+        self.unset_forced_down_hosts()
+        self.inspector.stop()
+        self.monitor.stop()
+        self.consumer.stop()
+        self.installer.cleanup()
         self.alarm.delete()
         self.instance.delete()
         self.network.delete()
         self.image.delete()
-        self.inspector.stop()
+        self.fault.cleanup()
         self.user.delete()
-        self.monitor.stop()
-        self.consumer.stop()
-        self.installer.cleanup()
 
 
 def main():