7 files changed, 228 insertions, 3 deletions
diff --git a/tests/config.py b/tests/config.py
index 94b85f7f..969d829f 100644
--- a/tests/config.py
+++ b/tests/config.py
@@ -15,6 +15,7 @@ import consumer
 import image
 import instance
 import network
+import inspector
 import os_clients
 import user
 
@@ -22,6 +23,7 @@ import user
 def list_opts():
     return [
         ('consumer', consumer.OPTS),
+        ('inspector', inspector.OPTS),
         ('DEFAULT', itertools.chain(
             os_clients.OPTS,
             image.OPTS,
diff --git a/tests/inspector.py b/tests/inspector.py
index d11da299..a61051f1 100644
--- a/tests/inspector.py
+++ b/tests/inspector.py
@@ -54,8 +54,7 @@ class DoctorInspectorSample(object):
         # Pool of novaclients for redundant usage
         for i in range(self.NUMBER_OF_CLIENTS):
             self.novaclients.append(
-                novaclient.Client(self.NOVA_API_VERSION, session=sess,
-                                  connection_pool=True))
+                novaclient.Client(self.NOVA_API_VERSION, session=sess))
         # Normally we use this client for non redundant API calls
         self.nova=self.novaclients[0]
         self.nova.servers.list(detailed=False)
diff --git a/tests/inspector/__init__.py b/tests/inspector/__init__.py
new file mode 100644
index 00000000..35bdb5b9
--- /dev/null
+++ b/tests/inspector/__init__.py
@@ -0,0 +1,37 @@
+#############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+    cfg.StrOpt('type',
+               default=os.environ.get('INSPECTOR_TYPE', 'sample'),
+               choices=['sample', 'congress', 'vitrage'],
+               help='the component of doctor inspector',
+               required=True),
+    cfg.StrOpt('ip',
+               default='127.0.0.1',
+               help='the ip of default inspector',
+               required=False),
+    cfg.StrOpt('port',
+               default='12345',
+               help='the port of default for inspector',
+               required=False),
+]
+
+
+_inspector_name_class_mapping = {
+    'sample': 'inspector.sample.SampleInspector',
+}
+
+def get_inspector(conf, log):
+    inspector_class = _inspector_name_class_mapping[conf.inspector.type]
+    return importutils.import_object(inspector_class, conf, log)
diff --git a/tests/inspector/base.py b/tests/inspector/base.py
new file mode 100644
index 00000000..854f0695
--- /dev/null
+++ b/tests/inspector/base.py
@@ -0,0 +1,30 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInspector(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+
+    @abc.abstractmethod
+    def get_inspector_url(self):
+        pass
+
+    @abc.abstractmethod
+    def start(self):
+        pass
+
+    @abc.abstractmethod
+    def stop(self):
+        pass
+\ No newline at end of file
diff --git a/tests/inspector/sample.py b/tests/inspector/sample.py
new file mode 100644
index 00000000..db477de9
--- /dev/null
+++ b/tests/inspector/sample.py
@@ -0,0 +1,151 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import collections
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import nova_client
+from inspector.base import BaseInspector
+
+
+class SampleInspector(BaseInspector):
+    event_type = 'compute.host.down'
+
+    def __init__(self, conf, log):
+        super(SampleInspector, self).__init__(conf, log)
+        self.inspector_url = self.get_inspector_url()
+        self.novaclients = list()
+        self._init_novaclients()
+        # Normally we use this client for non redundant API calls
+        self.nova = self.novaclients[0]
+
+        self.servers = collections.defaultdict(list)
+        self.hostnames = list()
+        self.app = None
+
+    def _init_novaclients(self):
+        self.NUMBER_OF_CLIENTS = self.conf.instance_count
+        auth = get_identity_auth(project=self.conf.doctor_project)
+        session = get_session(auth=auth)
+        for i in range(self.NUMBER_OF_CLIENTS):
+            self.novaclients.append(
+                nova_client(self.conf.nova_version, session))
+
+    def _init_servers_list(self):
+        self.servers.clear()
+        opts = {'all_tenants': True}
+        servers = self.nova.servers.list(search_opts=opts)
+        for server in servers:
+            try:
+                host = server.__dict__.get('OS-EXT-SRV-ATTR:host')
+                self.servers[host].append(server)
+                self.log.debug('get hostname=%s from server=%s' % (host, server))
+            except Exception as e:
+                self.log.info('can not get hostname from server=%s' % server)
+
+    def get_inspector_url(self):
+        return 'http://%s:%s' % (self.conf.inspector.ip, self.conf.inspector.port)
+
+    def start(self):
+        self.log.info('sample inspector start......')
+        self._init_servers_list()
+        self.app = InspectorApp(self.conf.inspector.port, self, self.log)
+        self.app.start()
+
+    def stop(self):
+        self.log.info('sample inspector stop......')
+        if not self.app:
+            return
+        for hostname in self.hostnames:
+            self.nova.services.force_down(hostname, 'nova-compute', False)
+
+        headers = {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+        }
+        url = '%s%s' % (self.inspector_url, 'shutdown') \
+            if self.inspector_url.endswith('/') else \
+            '%s%s' % (self.inspector_url, '/shutdown')
+        requests.post(url, data='', headers=headers)
+
+    def handle_events(self, events):
+        for event in events:
+            hostname = event['details']['hostname']
+            event_type = event['type']
+            if event_type == self.event_type:
+                self.hostnames.append(hostname)
+                self.disable_compute_host(hostname)
+
+    def disable_compute_host(self, hostname):
+        threads = []
+        if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS:
+            # TODO(tojuvone): This could be enhanced in future with dynamic
+            # reuse of self.novaclients when all threads in use
+            self.log.error('%d servers in %s. Can handle only %d'%(
+                           self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS))
+        for nova, server in zip(self.novaclients, self.servers[hostname]):
+            t = ThreadedResetState(nova, "error", server, self.log)
+            t.start()
+            threads.append(t)
+        for t in threads:
+            t.join()
+        self.nova.services.force_down(hostname, 'nova-compute', True)
+        self.log.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
+
+
+class ThreadedResetState(Thread):
+
+    def __init__(self, nova, state, server, log):
+        Thread.__init__(self)
+        self.nova = nova
+        self.state = state
+        self.server = server
+        self.log = log
+
+    def run(self):
+        self.nova.servers.reset_state(self.server, self.state)
+        self.log.info('doctor mark vm(%s) error at %s' % (self.server, time.time()))
+
+
+class InspectorApp(Thread):
+
+    def __init__(self, port, inspector, log):
+        Thread.__init__(self)
+        self.port = port
+        self.inspector = inspector
+        self.log = log
+
+    def run(self):
+        app = Flask('inspector')
+
+        @app.route('/events', methods=['PUT'])
+        def event_posted():
+            self.log.info('event posted in sample inspector at %s' % time.time())
+            self.log.info('sample inspector = %s' % self.inspector)
+            self.log.info('sample inspector received data = %s' % request.data)
+            events = json.loads(request.data)
+            self.inspector.handle_events(events)
+            return "OK"
+
+        @app.route('/shutdown', methods=['POST'])
+        def shutdown():
+            self.log.info('shutdown inspector app server at %s' % time.time())
+            func = request.environ.get('werkzeug.server.shutdown')
+            if func is None:
+                raise RuntimeError('Not running with the Werkzeug Server')
+            func()
+            return 'inspector app shutting down...'
+
+        app.run(host="0.0.0.0", port=self.port)
diff --git a/tests/main.py b/tests/main.py
index 1cd2b1d2..797e28b2 100644
--- a/tests/main.py
+++ b/tests/main.py
@@ -14,6 +14,7 @@ from alarm import Alarm
 import config
 from image import Image
 from instance import Instance
+from inspector import get_inspector
 import logger as doctor_log
 from user import User
 from network import Network
@@ -30,6 +31,7 @@ class DoctorTest(object):
         self.network = Network(self.conf, LOG)
         self.instance = Instance(self.conf, LOG)
         self.alarm = Alarm(self.conf, LOG)
+        self.inspector = get_inspector(self.conf, LOG)
 
     def setup(self):
         # prepare the cloud env
@@ -49,6 +51,9 @@ class DoctorTest(object):
         # creating alarm...
         self.alarm.create()
 
+        # starting doctor sample components...
+        self.inspector.start()
+
     def run(self):
         """run doctor test"""
         try:
@@ -72,6 +77,7 @@ class DoctorTest(object):
         self.network.delete()
         self.image.delete()
         self.user.delete()
+        self.inspector.stop()
 
 
 def main():
diff --git a/tests/run.sh b/tests/run.sh
index 713e494b..5c922265 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -48,7 +48,7 @@ as_admin_user="--os-username admin --os-project-name $DOCTOR_PROJECT
 get_compute_host_info() {
     # get computer host info which first VM boot in as admin user
     COMPUTE_HOST=$(openstack $as_admin_user server show ${VM_BASENAME}1 |
-                   grep "OS-EXT-SRV-ATTR:host" | awk '{ print $4 }')
+                   grep "OS-EXT-SRV-ATTR:host " | awk '{ print $4 }')
     compute_host_in_undercloud=${COMPUTE_HOST%%.*}
     die_if_not_set $LINENO COMPUTE_HOST "Failed to get compute hostname"