fix package path and move files under doctor_tests

Change-Id: Ibde6a36c43064e5fbea1a0b7a9b49349c343e42f
Signed-off-by: Ryota MIBU <r-mibu@cq.jp.nec.com>

35 files changed, 2446 insertions, 0 deletions

diff --git a/doctor_tests/alarm.py b/doctor_tests/alarm.py
new file mode 100644
index 00000000..3b1aaf3f
--- /dev/null
+++ b/doctor_tests/alarm.py
@@ -0,0 +1,93 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import aodh_client
+from doctor_tests.os_clients import nova_client
+
+OPTS = [
+    cfg.StrOpt('alarm_basename',
+               default='doctor_alarm',
+               help='the base name of alarm',
+               required=True),
+]
+
+
+class Alarm(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+        self.auth = get_identity_auth(project=self.conf.doctor_project)
+        self.aodh = \
+            aodh_client(conf.aodh_version,
+                        get_session(auth=self.auth))
+        self.nova = \
+            nova_client(conf.nova_version,
+                        get_session(auth=self.auth))
+        self._init_alarm_name()
+
+    def _init_alarm_name(self):
+        self.alarm_names = []
+        for i in range(0, self.conf.instance_count):
+            alarm_name = '%s%d' % (self.conf.alarm_basename, i)
+            self.alarm_names.append(alarm_name)
+
+    def create(self):
+        self.log.info('alarm create start......')
+
+        alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()}
+        servers = \
+            {getattr(server, 'name'): server
+             for server in self.nova.servers.list()}
+
+        for i in range(0, self.conf.instance_count):
+            alarm_name = self.alarm_names[i]
+            if alarm_name in alarms:
+                continue;
+            vm_name = '%s%d' % (self.conf.instance_basename, i)
+            vm_id = getattr(servers[vm_name], 'id')
+            alarm_request = dict(
+                name=alarm_name,
+                description=u'VM failure',
+                enabled=True,
+                alarm_actions=[u'http://%s:%d/failure'
+                               % (self.conf.consumer.ip,
+                                  self.conf.consumer.port)],
+                repeat_actions=False,
+                severity=u'moderate',
+                type=u'event',
+                event_rule=dict(
+                    event_type=u'compute.instance.update',
+                    query=[
+                        dict(field=u'traits.instance_id',
+                             type='',
+                             op=u'eq',
+                             value=vm_id),
+                        dict(field=u'traits.state',
+                             type='',
+                             op=u'eq',
+                             value=u'error')]))
+            self.aodh.alarm.create(alarm_request)
+
+        self.log.info('alarm create end......')
+
+    def delete(self):
+        self.log.info('alarm delete start.......')
+
+        alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()}
+        for alarm_name in self.alarm_names:
+            if alarm_name in alarms:
+                self.aodh.alarm.delete(alarms[alarm_name]['alarm_id'])
+
+        del self.alarm_names[:]
+
+        self.log.info('alarm delete end.......')
diff --git a/doctor_tests/common/__init__.py b/doctor_tests/common/__init__.py
new file mode 100644
index 00000000..e68a3070
--- /dev/null
+++ b/doctor_tests/common/__init__.py
@@ -0,0 +1,8 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
\ No newline at end of file
diff --git a/doctor_tests/common/constants.py b/doctor_tests/common/constants.py
new file mode 100644
index 00000000..72d037af
--- /dev/null
+++ b/doctor_tests/common/constants.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from collections import namedtuple
+
+
+Host = namedtuple('Host', ['name', 'ip'])
diff --git a/doctor_tests/common/utils.py b/doctor_tests/common/utils.py
new file mode 100644
index 00000000..2e823acb
--- /dev/null
+++ b/doctor_tests/common/utils.py
@@ -0,0 +1,105 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import json
+import os
+import paramiko
+import re
+
+
+def load_json_file(full_path):
+    """Loads JSON from file
+    :param target_filename:
+    :return:
+    """
+    if not os.path.isfile(full_path):
+        raise Exception('File(%s) does not exist' % full_path)
+
+    with open(full_path, 'r') as file:
+        return json.load(file)
+
+
+def write_json_file(full_path, data):
+    """write JSON from file
+    :param target_filename:
+    :return:
+    """
+
+    with open(full_path, 'w+') as file:
+        file.write(json.dumps(data))
+
+
+def match_rep_in_file(regex, full_path):
+    if not os.path.isfile(full_path):
+        raise Exception('File(%s) does not exist' % full_path)
+
+    with open(full_path, 'r') as file:
+        for line in file:
+            result = re.search(regex, line)
+            if result:
+                return result
+
+    return None
+
+
+class SSHClient(object):
+    def __init__(self, ip, username, password=None, pkey=None,
+                 key_filename=None, log=None, look_for_keys=False,
+                 allow_agent=False):
+        self.client = paramiko.SSHClient()
+        self.client.load_system_host_keys()
+        self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.client.connect(ip, username=username, password=password,
+                            pkey=pkey, key_filename=key_filename,
+                            look_for_keys=look_for_keys,
+                            allow_agent=allow_agent)
+        self.log = log
+
+    def __del__(self):
+        self.client.close()
+
+    def ssh(self, command):
+        if self.log:
+            self.log.info("Executing: %s" % command)
+        stdin, stdout, stderr = self.client.exec_command(command)
+        ret = stdout.channel.recv_exit_status()
+        output = list()
+        for line in stdout.read().splitlines():
+            output.append(line.decode('utf-8'))
+        if ret:
+            if self.log:
+                self.log.info("*** FAILED to run command %s (%s)" % (command, ret))
+            raise Exception(
+                "Unable to run \ncommand: %s\nret: %s"
+                % (command, ret))
+        if self.log:
+            self.log.info("*** SUCCESSFULLY run command %s" % command)
+        return ret, output
+
+    def scp(self, source, dest, method='put'):
+        if self.log:
+            self.log.info("Copy %s -> %s" % (source, dest))
+        ftp = self.client.open_sftp()
+        if method == 'put':
+            ftp.put(source, dest)
+        elif method == 'get':
+            ftp.get(source, dest)
+        ftp.close()
+
+
+def run_async(func):
+    from threading import Thread
+    from functools import wraps
+
+    @wraps(func)
+    def async_func(*args, **kwargs):
+        thread = Thread(target=func, args=args, kwargs=kwargs)
+        thread.start()
+        return thread
+
+    return async_func
diff --git a/doctor_tests/config.py b/doctor_tests/config.py
new file mode 100644
index 00000000..273e84d5
--- /dev/null
+++ b/doctor_tests/config.py
@@ -0,0 +1,54 @@
+##############################################################################

+# Copyright (c) 2017 ZTE Corporation and others.

+#

+# All rights reserved. This program and the accompanying materials

+# are made available under the terms of the Apache License, Version 2.0

+# which accompanies this distribution, and is available at

+# http://www.apache.org/licenses/LICENSE-2.0

+##############################################################################

+import itertools

+

+from oslo_config import cfg

+

+from doctor_tests import alarm

+from doctor_tests import consumer

+from doctor_tests import image

+from doctor_tests import instance

+from doctor_tests import installer

+from doctor_tests import network

+from doctor_tests import inspector

+from doctor_tests import monitor

+from doctor_tests import os_clients

+from doctor_tests import profiler_poc

+from doctor_tests import user

+

+

+def list_opts():

+    return [

+        ('installer', installer.OPTS),

+        ('monitor', monitor.OPTS),

+        ('inspector', inspector.OPTS),

+        ('consumer', consumer.OPTS),

+        ('DEFAULT', itertools.chain(

+            os_clients.OPTS,

+            image.OPTS,

+            user.OPTS,

+            network.OPTS,

+            instance.OPTS,

+            alarm.OPTS,

+            profiler_poc.OPTS))

+    ]

+

+

+def prepare_conf(args=None, conf=None, config_files=None):

+    if conf is None:

+        conf = cfg.ConfigOpts()

+

+    for group, options in list_opts():

+        conf.register_opts(list(options),

+                           group=None if group == 'DEFAULT' else group)

+

+    conf(args, project='doctor', validate_default_values=True,

+         default_config_files=config_files)

+

+    return conf

diff --git a/doctor_tests/consumer/__init__.py b/doctor_tests/consumer/__init__.py
new file mode 100644
index 00000000..2c66a547
--- /dev/null
+++ b/doctor_tests/consumer/__init__.py
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+    cfg.StrOpt('type',
+               default='sample',
+               choices=['sample'],
+               help='the component of doctor consumer',
+               required=True),
+    cfg.StrOpt('ip',
+               default='127.0.0.1',
+               help='the ip of consumer',
+               required=True),
+    cfg.IntOpt('port',
+               default='12346',
+               help='the port of doctor consumer',
+               required=True),
+]
+
+
+_consumer_name_class_mapping = {
+    'sample': 'doctor_tests.consumer.sample.SampleConsumer'
+}
+
+
+def get_consumer(conf, log):
+    consumer_class = _consumer_name_class_mapping.get(conf.consumer.type)
+    return importutils.import_object(consumer_class, conf, log)
diff --git a/doctor_tests/consumer/base.py b/doctor_tests/consumer/base.py
new file mode 100644
index 00000000..35170748
--- /dev/null
+++ b/doctor_tests/consumer/base.py
@@ -0,0 +1,26 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseConsumer(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+
+    @abc.abstractmethod
+    def start(self):
+        pass
+
+    @abc.abstractmethod
+    def stop(self):
+        pass
\ No newline at end of file
diff --git a/doctor_tests/consumer/sample.py b/doctor_tests/consumer/sample.py
new file mode 100644
index 00000000..d76a764b
--- /dev/null
+++ b/doctor_tests/consumer/sample.py
@@ -0,0 +1,71 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from doctor_tests.consumer.base import BaseConsumer
+
+
+class SampleConsumer(BaseConsumer):
+
+    def __init__(self, conf, log):
+        super(SampleConsumer, self).__init__(conf, log)
+        self.app = None
+
+    def start(self):
+        self.log.info('sample consumer start......')
+        self.app = ConsumerApp(self.conf.consumer.port, self, self.log)
+        self.app.start()
+
+    def stop(self):
+        self.log.info('sample consumer stop......')
+        if not self.app:
+            return
+        headers = {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+        }
+        url = 'http://%s:%d/shutdown'\
+              % (self.conf.consumer.ip,
+                 self.conf.consumer.port)
+        requests.post(url, data='', headers=headers)
+
+
+class ConsumerApp(Thread):
+
+    def __init__(self, port, consumer, log):
+        Thread.__init__(self)
+        self.port = port
+        self.consumer = consumer
+        self.log = log
+
+    def run(self):
+        app = Flask('consumer')
+
+        @app.route('/failure', methods=['POST'])
+        def event_posted():
+            self.log.info('doctor consumer notified at %s' % time.time())
+            self.log.info('sample consumer received data = %s' % request.data)
+            data = json.loads(request.data.decode('utf8'))
+            return 'OK'
+
+        @app.route('/shutdown', methods=['POST'])
+        def shutdown():
+            self.log.info('shutdown consumer app server at %s' % time.time())
+            func = request.environ.get('werkzeug.server.shutdown')
+            if func is None:
+                raise RuntimeError('Not running with the Werkzeug Server')
+            func()
+            return 'consumer app shutting down...'
+
+        app.run(host="0.0.0.0", port=self.port)
diff --git a/doctor_tests/identity_auth.py b/doctor_tests/identity_auth.py
new file mode 100644
index 00000000..2586720c
--- /dev/null
+++ b/doctor_tests/identity_auth.py
@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from keystoneauth1 import loading
+from keystoneauth1 import session
+
+
+def get_identity_auth(username=None, password=None, project=None):
+    auth_url = os.environ['OS_AUTH_URL']
+    username = username or os.environ['OS_USERNAME']
+    password = password or os.environ['OS_PASSWORD']
+    user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default'
+    user_domain_id = os.environ.get('OS_USER_DOMAIN_ID') or 'default'
+    project_name = project or os.environ.get('OS_PROJECT_NAME') \
+                   or os.environ.get('OS_TENANT_NAME')
+    project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default'
+    project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID') or 'default'
+
+    loader = loading.get_plugin_loader('password')
+    return loader.load_from_options(
+        auth_url=auth_url,
+        username=username,
+        password=password,
+        user_domain_name=user_domain_name,
+        user_domain_id=user_domain_id,
+        project_name=project_name,
+        tenant_name=project_name,
+        project_domain_name=project_domain_name,
+        project_domain_id=project_domain_id)
+
+
+def get_session(auth=None):
+    """Get a user credentials auth session."""
+    if auth is None:
+        auth = get_identity_auth()
+    return session.Session(auth=auth)
diff --git a/doctor_tests/image.py b/doctor_tests/image.py
new file mode 100644
index 00000000..2e313e12
--- /dev/null
+++ b/doctor_tests/image.py
@@ -0,0 +1,74 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import urllib.request
+
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import glance_client
+
+OPTS = [
+    cfg.StrOpt('image_name',
+               default=os.environ.get('IMAGE_NAME', 'cirros'),
+               help='the name of test image',
+               required=True),
+    cfg.StrOpt('image_format',
+               default='qcow2',
+               help='the format of test image',
+               required=True),
+    cfg.StrOpt('image_filename',
+               default='cirros.img',
+               help='the name of image file',
+               required=True),
+    cfg.StrOpt('image_download_url',
+               default='https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img',
+               help='the url where to get the image',
+               required=True),
+]
+
+
+class Image(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+        self.glance = \
+            glance_client(conf.glance_version, get_session())
+        self.use_existing_image = False
+        self.image = None
+
+    def create(self):
+        self.log.info('image create start......')
+
+        images = {image.name: image for image in self.glance.images.list()}
+        if self.conf.image_name not in images:
+            if not os.path.exists(self.conf.image_filename):
+                resp = urllib.request.urlopen(self.conf.image_download_url)
+                with open(self.conf.image_filename, "wb") as file:
+                    file.write(resp.read())
+            self.image = self.glance.images.create(name=self.conf.image_name,
+                                                   disk_format=self.conf.image_format,
+                                                   container_format="bare",
+                                                   visibility="public")
+            self.glance.images.upload(self.image['id'],
+                                      open(self.conf.image_filename, 'rb'))
+        else:
+            self.use_existing_image = True
+            self.image = images[self.conf.image_name]
+
+        self.log.info('image create end......')
+
+    def delete(self):
+        self.log.info('image delete start.......')
+
+        if not self.use_existing_image and self.image:
+            self.glance.images.delete(self.image['id'])
+
+        self.log.info('image delete end.......')
diff --git a/doctor_tests/inspector/__init__.py b/doctor_tests/inspector/__init__.py
new file mode 100644
index 00000000..3be79e57
--- /dev/null
+++ b/doctor_tests/inspector/__init__.py
@@ -0,0 +1,40 @@
+#############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+    cfg.StrOpt('type',
+               default=os.environ.get('INSPECTOR_TYPE', 'sample'),
+               choices=['sample', 'congress', 'vitrage'],
+               help='the component of doctor inspector',
+               required=True),
+    cfg.StrOpt('ip',
+               default='127.0.0.1',
+               help='the host ip of inspector',
+               required=False),
+    cfg.StrOpt('port',
+               default='12345',
+               help='the port of default for inspector',
+               required=False),
+]
+
+
+_inspector_name_class_mapping = {
+    'sample': 'doctor_tests.inspector.sample.SampleInspector',
+    'congress': 'doctor_tests.inspector.congress.CongressInspector',
+}
+
+
+def get_inspector(conf, log):
+    inspector_class = _inspector_name_class_mapping[conf.inspector.type]
+    return importutils.import_object(inspector_class, conf, log)
diff --git a/doctor_tests/inspector/base.py b/doctor_tests/inspector/base.py
new file mode 100644
index 00000000..854f0695
--- /dev/null
+++ b/doctor_tests/inspector/base.py
@@ -0,0 +1,30 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInspector(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+
+    @abc.abstractmethod
+    def get_inspector_url(self):
+        pass
+
+    @abc.abstractmethod
+    def start(self):
+        pass
+
+    @abc.abstractmethod
+    def stop(self):
+        pass
\ No newline at end of file
diff --git a/doctor_tests/inspector/congress.py b/doctor_tests/inspector/congress.py
new file mode 100644
index 00000000..c89a41bd
--- /dev/null
+++ b/doctor_tests/inspector/congress.py
@@ -0,0 +1,94 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import congress_client
+
+from doctor_tests.inspector.base import BaseInspector
+
+
+class CongressInspector(BaseInspector):
+    nova_api_min_version = '2.11'
+    doctor_driver = 'doctor'
+    doctor_datasource = 'doctor'
+    policy = 'classification'
+    rules = {
+        'host_down':
+            'host_down(host) :- doctor:events(hostname=host, type="compute.host.down", status="down")',
+        'active_instance_in_host':
+            'active_instance_in_host(vmid, host) :- nova:servers(id=vmid, host_name=host, status="ACTIVE")',
+        'host_force_down':
+            'execute[nova:services.force_down(host, "nova-compute", "True")] :- host_down(host)',
+        'error_vm_states':
+            'execute[nova:servers.reset_state(vmid, "error")] :- host_down(host), active_instance_in_host(vmid, host)'
+    }
+
+    def __init__(self, conf, log):
+        super(CongressInspector, self).__init__(conf, log)
+        self.auth = get_identity_auth()
+        self.congress = congress_client(get_session(auth=self.auth))
+        self._init_driver_and_ds()
+        self.inspector_url = self.get_inspector_url()
+
+    def _init_driver_and_ds(self):
+        datasources = \
+            {ds['name']: ds for ds in self.congress.list_datasources()['results']}
+
+        # check nova_api version
+        nova_api_version = datasources['nova']['config'].get('api_version')
+        if nova_api_version and nova_api_version < self.nova_api_min_version:
+            raise Exception('Congress Nova datasource API version < nova_api_min_version(%s)'
+                            % self.nova_api_min_version)
+
+        # create doctor datasource if it's not exist
+        if self.doctor_datasource not in datasources:
+            self.congress.create_datasource(
+                body={'driver': self.doctor_driver,
+                      'name': self.doctor_datasource})
+
+        # check whether doctor driver exist
+        drivers = \
+            {driver['id']: driver for driver in self.congress.list_drivers()['results']}
+        if self.doctor_driver not in drivers:
+            raise Exception('Do not support doctor driver in congress')
+
+        self.policy_rules = \
+            {rule['name']: rule for rule in
+             self.congress.list_policy_rules(self.policy)['results']}
+
+    def get_inspector_url(self):
+        ds = self.congress.list_datasources()['results']
+        doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+                         None)
+        congress_endpoint = self.congress.httpclient.get_endpoint(auth=self.auth)
+        return ('%s/v1/data-sources/%s/tables/events/rows' %
+                (congress_endpoint, doctor_ds['id']))
+
+    def start(self):
+        self.log.info('congress inspector start......')
+
+        for rule_name, rule in self.rules.items():
+            self._add_rule(rule_name, rule)
+
+    def stop(self):
+        self.log.info('congress inspector stop......')
+
+        for rule_name in self.rules.keys():
+            self._del_rule(rule_name)
+
+    def _add_rule(self, rule_name, rule):
+        if rule_name not in self.policy_rules:
+            self.congress.create_policy_rule(self.policy,
+                                             body={'name': rule_name,
+                                                   'rule': rule})
+
+    def _del_rule(self, rule_name):
+        if rule_name in self.policy_rules:
+            rule_id = self.policy_rules[rule_name]['id']
+            self.congress.delete_policy_rule(self.policy, rule_id)
diff --git a/doctor_tests/inspector/sample.py b/doctor_tests/inspector/sample.py
new file mode 100644
index 00000000..114e4ebd
--- /dev/null
+++ b/doctor_tests/inspector/sample.py
@@ -0,0 +1,169 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import collections
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from doctor_tests.common import utils
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import nova_client
+from doctor_tests.os_clients import neutron_client
+from doctor_tests.inspector.base import BaseInspector
+
+
+class SampleInspector(BaseInspector):
+    event_type = 'compute.host.down'
+
+    def __init__(self, conf, log):
+        super(SampleInspector, self).__init__(conf, log)
+        self.inspector_url = self.get_inspector_url()
+        self.novaclients = list()
+        self._init_novaclients()
+        # Normally we use this client for non redundant API calls
+        self.nova = self.novaclients[0]
+
+        auth = get_identity_auth(project=self.conf.doctor_project)
+        session = get_session(auth=auth)
+        self.neutron = neutron_client(session)
+
+        self.servers = collections.defaultdict(list)
+        self.hostnames = list()
+        self.app = None
+
+    def _init_novaclients(self):
+        self.NUMBER_OF_CLIENTS = self.conf.instance_count
+        auth = get_identity_auth(project=self.conf.doctor_project)
+        session = get_session(auth=auth)
+        for i in range(self.NUMBER_OF_CLIENTS):
+            self.novaclients.append(
+                nova_client(self.conf.nova_version, session))
+
+    def _init_servers_list(self):
+        self.servers.clear()
+        opts = {'all_tenants': True}
+        servers = self.nova.servers.list(search_opts=opts)
+        for server in servers:
+            try:
+                host = server.__dict__.get('OS-EXT-SRV-ATTR:host')
+                self.servers[host].append(server)
+                self.log.debug('get hostname=%s from server=%s' % (host, server))
+            except Exception as e:
+                self.log.info('can not get hostname from server=%s' % server)
+
+    def get_inspector_url(self):
+        return 'http://%s:%s' % (self.conf.inspector.ip, self.conf.inspector.port)
+
+    def start(self):
+        self.log.info('sample inspector start......')
+        self._init_servers_list()
+        self.app = InspectorApp(self.conf.inspector.port, self, self.log)
+        self.app.start()
+
+    def stop(self):
+        self.log.info('sample inspector stop......')
+        if not self.app:
+            return
+        for hostname in self.hostnames:
+            self.nova.services.force_down(hostname, 'nova-compute', False)
+
+        headers = {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+        }
+        url = '%s%s' % (self.inspector_url, 'shutdown') \
+            if self.inspector_url.endswith('/') else \
+            '%s%s' % (self.inspector_url, '/shutdown')
+        requests.post(url, data='', headers=headers)
+
+    def handle_events(self, events):
+        for event in events:
+            hostname = event['details']['hostname']
+            event_type = event['type']
+            if event_type == self.event_type:
+                self.hostnames.append(hostname)
+                thr1 = self._disable_compute_host(hostname)
+                thr2 = self._vms_reset_state('error', hostname)
+                thr3 = self._set_ports_data_plane_status('DOWN', hostname)
+                thr1.join()
+                thr2.join()
+                thr3.join()
+
+    @utils.run_async
+    def _disable_compute_host(self, hostname):
+        self.nova.services.force_down(hostname, 'nova-compute', True)
+        self.log.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
+
+    @utils.run_async
+    def _vms_reset_state(self, state, hostname):
+
+        @utils.run_async
+        def _vm_reset_state(nova, server, state):
+            nova.servers.reset_state(server, state)
+            self.log.info('doctor mark vm(%s) error at %s' % (server, time.time()))
+
+        thrs = []
+        for nova, server in zip(self.novaclients, self.servers[hostname]):
+            t = _vm_reset_state(nova, server, state)
+            thrs.append(t)
+        for t in thrs:
+            t.join()
+
+    @utils.run_async
+    def _set_ports_data_plane_status(self, status, hostname):
+        body = {'data_plane_status': status}
+
+        @utils.run_async
+        def _set_port_data_plane_status(port_id):
+            self.neutron.update_port(port_id, body)
+            self.log.info('doctor set data plane status %s on port %s' % (status, port_id))
+
+        thrs = []
+        params = {'binding:host_id': hostname}
+        for port_id in self.neutron.list_ports(**params):
+            t = _set_port_data_plane_status(port_id)
+            thrs.append(t)
+        for t in thrs:
+            t.join()
+
+
+class InspectorApp(Thread):
+
+    def __init__(self, port, inspector, log):
+        Thread.__init__(self)
+        self.port = port
+        self.inspector = inspector
+        self.log = log
+
+    def run(self):
+        app = Flask('inspector')
+
+        @app.route('/events', methods=['PUT'])
+        def event_posted():
+            self.log.info('event posted in sample inspector at %s' % time.time())
+            self.log.info('sample inspector = %s' % self.inspector)
+            self.log.info('sample inspector received data = %s' % request.data)
+            events = json.loads(request.data.decode('utf8'))
+            self.inspector.handle_events(events)
+            return "OK"
+
+        @app.route('/shutdown', methods=['POST'])
+        def shutdown():
+            self.log.info('shutdown inspector app server at %s' % time.time())
+            func = request.environ.get('werkzeug.server.shutdown')
+            if func is None:
+                raise RuntimeError('Not running with the Werkzeug Server')
+            func()
+            return 'inspector app shutting down...'
+
+        app.run(host="0.0.0.0", port=self.port)
diff --git a/doctor_tests/installer/__init__.py b/doctor_tests/installer/__init__.py
new file mode 100644
index 00000000..02735b11
--- /dev/null
+++ b/doctor_tests/installer/__init__.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+    cfg.StrOpt('type',
+               default=os.environ.get('INSTALLER_TYPE', 'local'),
+               choices=['local', 'apex'],
+               help='the type of installer',
+               required=True),
+    cfg.StrOpt('ip',
+               default=os.environ.get('INSTALLER_IP', '127.0.0.1'),
+               help='the ip of installer'),
+    cfg.StrOpt('username',
+               default='root',
+               help='the user name for login installer server',
+               required=True),
+]
+
+
+_installer_name_class_mapping = {
+    'local': 'doctor_tests.installer.local.LocalInstaller',
+    'apex': 'doctor_tests.installer.apex.ApexInstaller'
+}
+
+
+def get_installer(conf, log):
+    installer_class = _installer_name_class_mapping[conf.installer.type]
+    return importutils.import_object(installer_class, conf, log)
diff --git a/doctor_tests/installer/apex.py b/doctor_tests/installer/apex.py
new file mode 100644
index 00000000..2a1ce94b
--- /dev/null
+++ b/doctor_tests/installer/apex.py
@@ -0,0 +1,126 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import getpass
+import grp
+import os
+import pwd
+import stat
+import subprocess
+
+from doctor_tests.common.utils import SSHClient
+from doctor_tests.installer.base import BaseInstaller
+
+
+class ApexInstaller(BaseInstaller):
+    node_user_name = 'heat-admin'
+    cm_set_script = 'set_ceilometer.py'
+    cm_restore_script = 'restore_ceilometer.py'
+
+    def __init__(self, conf, log):
+        super(ApexInstaller, self).__init__(conf, log)
+        self.client = SSHClient(self.conf.installer.ip,
+                                self.conf.installer.username,
+                                look_for_keys=True)
+        self.key_file = None
+        self.controllers = list()
+        self.controller_clients = list()
+        self.servers = list()
+
+    def setup(self):
+        self.log.info('Setup Apex installer start......')
+
+        self.get_ssh_key_from_installer()
+        self.get_controller_ips()
+        self.set_apply_patches()
+        self.setup_stunnel()
+
+    def cleanup(self):
+        self.restore_apply_patches()
+        for server in self.servers:
+            server.terminate()
+
+    def get_ssh_key_from_installer(self):
+        self.log.info('Get SSH keys from Apex installer......')
+
+        if self.key_file is not None:
+            self.log.info('Already have SSH keys from Apex installer......')
+            return self.key_file
+
+        self.client.scp('/home/stack/.ssh/id_rsa', './instack_key', method='get')
+        user = getpass.getuser()
+        uid = pwd.getpwnam(user).pw_uid
+        gid = grp.getgrnam(user).gr_gid
+        os.chown('./instack_key', uid, gid)
+        os.chmod('./instack_key', stat.S_IREAD)
+        current_dir = os.curdir
+        self.key_file = '{0}/{1}'.format(current_dir, 'instack_key')
+        return self.key_file
+
+    def get_controller_ips(self):
+        self.log.info('Get controller ips from Apex installer......')
+
+        command = "source stackrc; " \
+                  "nova list | grep ' overcloud-controller-[0-9] ' " \
+                  "| sed -e 's/^.*ctlplane=//' |awk '{print $1}'"
+        ret, controllers = self.client.ssh(command)
+        if ret:
+            raise Exception('Exec command to get controller ips in Apex installer failed'
+                            'ret=%s, output=%s' % (ret, controllers))
+        self.log.info('Get controller_ips:%s from Apex installer' % controllers)
+        self.controllers = controllers
+
+    def get_host_ip_from_hostname(self, hostname):
+        self.log.info('Get host ip from host name in Apex installer......')
+
+        hostname_in_undercloud = hostname.split('.')[0]
+
+        command = "source stackrc; nova show %s  | awk '/ ctlplane network /{print $5}'" % (hostname_in_undercloud)
+        ret, host_ip = self.client.ssh(command)
+        if ret:
+            raise Exception('Exec command to get host ip from hostname(%s) in Apex installer failed'
+                            'ret=%s, output=%s' % (hostname, ret, host_ip))
+        self.log.info('Get host_ip:%s from host_name:%s in Apex installer' % (host_ip, hostname))
+        return host_ip[0]
+
+    def setup_stunnel(self):
+        self.log.info('Setup ssh stunnel in controller nodes in Apex installer......')
+        for node_ip in self.controllers:
+            cmd = "sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s -R %s:localhost:%s sleep 600 > ssh_tunnel.%s.log 2>&1 < /dev/null &" \
+                  % (self.key_file, self.node_user_name, node_ip,
+                     self.conf.consumer.port, self.conf.consumer.port, node_ip)
+            server = subprocess.Popen(cmd, shell=True)
+            self.servers.append(server)
+            server.communicate()
+
+    def set_apply_patches(self):
+        self.log.info('Set apply patches start......')
+
+        for node_ip in self.controllers:
+            client = SSHClient(node_ip, self.node_user_name, key_filename=self.key_file)
+            self.controller_clients.append(client)
+            self._ceilometer_apply_patches(client, self.cm_set_script)
+
+    def restore_apply_patches(self):
+        self.log.info('restore apply patches start......')
+
+        for client in self.controller_clients:
+            self._ceilometer_apply_patches(client, self.cm_restore_script)
+
+    def _ceilometer_apply_patches(self, ssh_client, script_name):
+        installer_dir = os.path.dirname(os.path.realpath(__file__))
+        script_abs_path = '{0}/{1}/{2}'.format(installer_dir, 'common', script_name)
+
+        ssh_client.scp(script_abs_path, script_name)
+        cmd = 'sudo python %s' % script_name
+        ret, output = ssh_client.ssh(cmd)
+        if ret:
+            raise Exception('Do the ceilometer command in controller node failed....'
+                            'ret=%s, cmd=%s, output=%s' % (ret, cmd, output))
+        ssh_client.ssh('sudo systemctl restart openstack-ceilometer-notification.service')
+
diff --git a/doctor_tests/installer/base.py b/doctor_tests/installer/base.py
new file mode 100644
index 00000000..fa39816a
--- /dev/null
+++ b/doctor_tests/installer/base.py
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInstaller(object):
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+
+    @abc.abstractproperty
+    def node_user_name(self):
+        """user name for login to cloud node"""
+
+    @abc.abstractmethod
+    def get_ssh_key_from_installer(self):
+        pass
+
+    @abc.abstractmethod
+    def get_host_ip_from_hostname(self, hostname):
+        pass
+
+    @abc.abstractmethod
+    def setup(self):
+        pass
+
+    @abc.abstractmethod
+    def cleanup(self):
+        pass
diff --git a/doctor_tests/installer/common/congress.py b/doctor_tests/installer/common/congress.py
new file mode 100644
index 00000000..db882de2
--- /dev/null
+++ b/doctor_tests/installer/common/congress.py
@@ -0,0 +1,47 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+def set_doctor_driver_conf(ssh_client, restart_cmd):
+    cg_set_cmd = '''#!/bin/bash
+co_conf=/etc/congress/congress.conf
+co_conf_bak=/etc/congress/congress.conf.bak
+co_entry="congress.datasources.doctor_driver.DoctorDriver"
+if sudo grep -e "^drivers.*$co_entry" $co_conf; then
+    echo "NOTE: congress is configured as we needed"
+else
+    echo "modify the congress config"
+    sudo cp $co_conf $co_conf_bak
+    sudo sed -i -e "/^drivers/s/$/,$co_entry/"  $co_conf
+    %s
+fi
+    ''' % (restart_cmd)
+
+    ret, output = ssh_client.ssh(cg_set_cmd)
+    if ret:
+        raise Exception('Do the congress command in controller node failed....'
+                        'ret=%s, cmd=%s, output=%s' % (ret, cg_set_cmd, output))
+
+
+def restore_doctor_driver_conf(ssh_client, restart_cmd):
+    cg_restore_cmd = '''#!/bin/bash
+co_conf=/etc/congress/congress.conf
+co_conf_bak=/etc/congress/congress.conf.bak
+if [ -e $co_conf_bak ]; then
+    echo "restore the congress config"
+    sudo cp $co_conf_bak $co_conf
+    sudo rm $co_conf_bak
+    %s
+else
+    echo "Do not need to restore the congress config"
+fi
+    ''' % (restart_cmd)
+
+    ret, output = ssh_client.ssh(cg_restore_cmd)
+    if ret:
+        raise Exception('Do the congress command in controller node failed....'
+                        'ret=%s, cmd=%s, output=%s' % (ret, cg_restore_cmd, output))
diff --git a/doctor_tests/installer/common/restore_ceilometer.py b/doctor_tests/installer/common/restore_ceilometer.py
new file mode 100644
index 00000000..d25b9ede
--- /dev/null
+++ b/doctor_tests/installer/common/restore_ceilometer.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+
+
+def restore_ep_config():
+
+    if not os.path.isfile(ep_file_bak):
+        print('Bak_file:%s does not exist.' % ep_file_bak)
+    else:
+        print('restore')
+        shutil.copyfile(ep_file_bak, ep_file)
+        os.remove(ep_file_bak)
+    return
+
+
+restore_ep_config()
diff --git a/doctor_tests/installer/common/set_ceilometer.py b/doctor_tests/installer/common/set_ceilometer.py
new file mode 100644
index 00000000..f5946cb2
--- /dev/null
+++ b/doctor_tests/installer/common/set_ceilometer.py
@@ -0,0 +1,44 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+import yaml
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+event_notifier_topic = 'notifier://?topic=alarm.all'
+
+
+def set_notifier_topic():
+    config_modified = False
+
+    if not os.path.isfile(ep_file):
+        raise Exception("File doesn't exist: %s." % ep_file)
+
+    with open(ep_file, 'r') as file:
+        config = yaml.safe_load(file)
+
+    sinks = config['sinks']
+    for sink in sinks:
+        if sink['name'] == 'event_sink':
+            publishers = sink['publishers']
+            if event_notifier_topic not in publishers:
+                print('Add event notifier in ceilometer')
+                publishers.append(event_notifier_topic)
+                config_modified = True
+            else:
+                print('NOTE: event notifier is configured in ceilometer as we needed')
+
+    if config_modified:
+        shutil.copyfile(ep_file, ep_file_bak)
+        with open(ep_file, 'w+') as file:
+            file.write(yaml.safe_dump(config))
+
+
+set_notifier_topic()
diff --git a/doctor_tests/installer/local.py b/doctor_tests/installer/local.py
new file mode 100644
index 00000000..7d0ae542
--- /dev/null
+++ b/doctor_tests/installer/local.py
@@ -0,0 +1,109 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+import subprocess
+
+from doctor_tests.installer.base import BaseInstaller
+from doctor_tests.common.utils import load_json_file
+from doctor_tests.common.utils import write_json_file
+
+
+class LocalInstaller(BaseInstaller):
+    node_user_name = 'root'
+
+    nova_policy_file = '/etc/nova/policy.json'
+    nova_policy_file_backup = '%s%s' % (nova_policy_file, '.bak')
+
+    def __init__(self, conf, log):
+        super(LocalInstaller, self).__init__(conf, log)
+        self.policy_modified = False
+        self.add_policy_file = False
+
+    def setup(self):
+        self.get_ssh_key_from_installer()
+        self.set_apply_patches()
+
+    def cleanup(self):
+        self.restore_apply_patches()
+
+    def get_ssh_key_from_installer(self):
+        self.log.info('Assuming SSH keys already exchanged with computer for local installer type')
+        return None
+
+    def get_host_ip_from_hostname(self, hostname):
+        self.log.info('Get host ip from host name in local installer......')
+
+        cmd = "getent hosts %s | awk '{ print $1 }'" % (hostname)
+        server = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+        stdout, stderr = server.communicate()
+        host_ip = stdout.strip()
+
+        self.log.info('Get host_ip:%s from host_name:%s in local installer' % (host_ip, hostname))
+        return host_ip
+
+    def set_apply_patches(self):
+        self._set_nova_policy()
+
+    def restore_apply_patches(self):
+        self._restore_nova_policy()
+
+    def _set_nova_policy(self):
+        host_status_policy = 'os_compute_api:servers:show:host_status'
+        host_status_rule = 'rule:admin_or_owner'
+        policy_data = {
+            'context_is_admin': 'role:admin',
+            'owner': 'user_id:%(user_id)s',
+            'admin_or_owner': 'rule:context_is_admin or rule:owner',
+            host_status_policy: host_status_rule
+        }
+
+        if os.path.isfile(self.nova_policy_file):
+            data = load_json_file(self.nova_policy_file)
+            if host_status_policy in data:
+                rule_origion = data[host_status_policy]
+                if host_status_rule == rule_origion:
+                    self.log.info('Do not need to modify nova policy.')
+                    self.policy_modified = False
+                else:
+                    # update the host_status_policy
+                    data[host_status_policy] = host_status_rule
+                    self.policy_modified = True
+            else:
+                # add the host_status_policy, if the admin_or_owner is not
+                # defined, add it also
+                for policy, rule in policy_data.items():
+                    if policy not in data:
+                        data[policy] = rule
+                self.policy_modified = True
+            if self.policy_modified:
+                self.log.info('Nova policy is Modified.')
+                shutil.copyfile(self.nova_policy_file,
+                                self.nova_policy_file_backup)
+        else:
+            # file does not exit, create a new one and add the policy
+            self.log.info('Nova policy file not exist. Creating a new one')
+            data = policy_data
+            self.add_policy_file = True
+
+        if self.policy_modified or self.add_policy_file:
+            write_json_file(self.nova_policy_file, data)
+            os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"')
+
+    def _restore_nova_policy(self):
+        if self.policy_modified:
+            shutil.copyfile(self.nova_policy_file_backup, self.nova_policy_file)
+            os.remove(self.nova_policy_file_backup)
+        elif self.add_policy_file:
+            os.remove(self.nova_policy_file)
+
+        if self.add_policy_file or self.policy_modified:
+            os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"')
+            self.add_policy_file = False
+            self.policy_modified = False
diff --git a/doctor_tests/instance.py b/doctor_tests/instance.py
new file mode 100644
index 00000000..27f412e2
--- /dev/null
+++ b/doctor_tests/instance.py
@@ -0,0 +1,114 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import time
+
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import neutron_client
+from doctor_tests.os_clients import nova_client
+
+OPTS = [
+    cfg.StrOpt('flavor',
+               default='m1.tiny',
+               help='the name of flavor',
+               required=True),
+    cfg.IntOpt('instance_count',
+               default=os.environ.get('VM_COUNT', 1),
+               help='the count of instance',
+               required=True),
+    cfg.StrOpt('instance_basename',
+               default='doctor_vm',
+               help='the base name of instance',
+               required=True),
+]
+
+
+class Instance(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+        self.auth = get_identity_auth(username=self.conf.doctor_user,
+                                      password=self.conf.doctor_passwd,
+                                      project=self.conf.doctor_project)
+        self.nova = \
+            nova_client(conf.nova_version,
+                        get_session(auth=self.auth))
+        self.neutron = neutron_client(get_session(auth=self.auth))
+        self.servers = {}
+        self.vm_names = []
+
+    def create(self):
+        self.log.info('instance create start......')
+
+        # get flavor, image and network for vm boot
+        flavors = {flavor.name: flavor for flavor in self.nova.flavors.list()}
+        flavor = flavors.get(self.conf.flavor)
+        image = self.nova.glance.find_image(self.conf.image_name)
+        network = self.neutron.list_networks(name=self.conf.net_name)['networks'][0]
+        nics = {'net-id': network['id']}
+
+        self.servers = \
+            {getattr(server, 'name'): server
+             for server in self.nova.servers.list()}
+        for i in range(0, self.conf.instance_count):
+            vm_name = "%s%d"%(self.conf.instance_basename, i)
+            self.vm_names.append(vm_name)
+            if vm_name not in self.servers:
+                server = self.nova.servers.create(vm_name, image,
+                                                  flavor, nics=[nics])
+                self.servers[vm_name] = server
+                time.sleep(0.1)
+
+        self.log.info('instance create end......')
+
+    def delete(self):
+        self.log.info('instance delete start.......')
+
+        for vm_name in self.vm_names:
+            if vm_name in self.servers:
+                self.nova.servers.delete(self.servers[vm_name])
+                time.sleep(0.1)
+
+        # check that all vms are deleted
+        while self.nova.servers.list():
+            time.sleep(0.1)
+        self.servers.clear()
+        del self.vm_names[:]
+
+        self.log.info('instance delete end.......')
+
+    def wait_for_vm_launch(self):
+        self.log.info('wait for vm launch start......')
+
+        wait_time = 60
+        count = 0
+        while count < wait_time:
+            active_count = 0
+            for vm_name in self.vm_names:
+                server = self.nova.servers.get(self.servers[vm_name])
+                server_status = getattr(server, 'status').lower()
+                if 'active' == server_status:
+                    active_count += 1
+                elif 'error' == server_status:
+                    raise Exception('vm launched with error state')
+                else:
+                    time.sleep(2)
+                    count += 1
+                    continue
+            if active_count == self.conf.instance_count:
+                self.log.info('wait for vm launch end......')
+                return
+            count += 1
+            time.sleep(2)
+        raise Exception('time out for vm launch')
+
diff --git a/doctor_tests/logger.py b/doctor_tests/logger.py
new file mode 100644
index 00000000..b7a49fdb
--- /dev/null
+++ b/doctor_tests/logger.py
@@ -0,0 +1,46 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+# Usage:
+#  import doctor_logger
+#  logger = doctor_logger.Logger("script_name").getLogger()
+#  logger.info("message to be shown with - INFO - ")
+#  logger.debug("message to be shown with - DEBUG -")
+
+import logging
+import os
+
+
+class Logger(object):
+    def __init__(self, logger_name):
+
+        CI_DEBUG = os.getenv('CI_DEBUG')
+
+        self.logger = logging.getLogger(logger_name)
+        self.logger.propagate = 0
+        self.logger.setLevel(logging.DEBUG)
+
+        formatter = logging.Formatter('%(asctime)s %(filename)s %(lineno)d '
+                                      '%(levelname)-6s %(message)s')
+
+        ch = logging.StreamHandler()
+        ch.setFormatter(formatter)
+        if CI_DEBUG is not None and CI_DEBUG.lower() == "true":
+            ch.setLevel(logging.DEBUG)
+        else:
+            ch.setLevel(logging.INFO)
+        self.logger.addHandler(ch)
+
+        filename = '%s.log' % logger_name
+        file_handler = logging.FileHandler(filename, mode='w')
+        file_handler.setFormatter(formatter)
+        file_handler.setLevel(logging.DEBUG)
+        self.logger.addHandler(file_handler)
+
+    def getLogger(self):
+        return self.logger
diff --git a/doctor_tests/main.py b/doctor_tests/main.py
new file mode 100644
index 00000000..006aac9f
--- /dev/null
+++ b/doctor_tests/main.py
@@ -0,0 +1,215 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+from os.path import isfile, join
+import random
+import sys
+import time
+
+from doctor_tests.alarm import Alarm
+from doctor_tests.common.constants import Host
+from doctor_tests.common.utils import match_rep_in_file
+from doctor_tests import config
+from doctor_tests.consumer import get_consumer
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.image import Image
+from doctor_tests.instance import Instance
+from doctor_tests.inspector import get_inspector
+from doctor_tests.installer import get_installer
+import doctor_tests.logger as doctor_log
+from doctor_tests.network import Network
+from doctor_tests.monitor import get_monitor
+from doctor_tests.os_clients import nova_client
+from doctor_tests.profiler_poc import main as profiler_main
+from doctor_tests.scenario.common import calculate_notification_time
+from doctor_tests.scenario.network_failure import NetworkFault
+from doctor_tests.user import User
+
+
+LOG = doctor_log.Logger('doctor').getLogger()
+
+
+class DoctorTest(object):
+
+    def __init__(self, conf):
+        self.conf = conf
+        self.image = Image(self.conf, LOG)
+        self.user = User(self.conf, LOG)
+        self.network = Network(self.conf, LOG)
+        self.instance = Instance(self.conf, LOG)
+        self.alarm = Alarm(self.conf, LOG)
+        self.installer = get_installer(self.conf, LOG)
+        self.inspector = get_inspector(self.conf, LOG)
+        self.monitor = get_monitor(self.conf,
+                                   self.inspector.get_inspector_url(),
+                                   LOG)
+        self.consumer = get_consumer(self.conf, LOG)
+        self.fault = NetworkFault(self.conf, self.installer, LOG)
+        auth = get_identity_auth(project=self.conf.doctor_project)
+        self.nova = nova_client(self.conf.nova_version,
+                                get_session(auth=auth))
+        self.down_host = None
+
+    def setup(self):
+        # prepare the cloud env
+        self.installer.setup()
+
+        # preparing VM image...
+        self.image.create()
+
+        # creating test user...
+        self.user.create()
+        self.user.update_quota()
+
+        # creating VM...
+        self.network.create()
+        self.instance.create()
+        self.instance.wait_for_vm_launch()
+
+        # creating alarm...
+        self.alarm.create()
+
+        # starting doctor sample components...
+        self.inspector.start()
+
+        self.down_host = self.get_host_info_for_random_vm()
+        self.monitor.start(self.down_host)
+
+        self.consumer.start()
+
+    def run(self):
+        """run doctor test"""
+        try:
+            LOG.info('doctor test starting.......')
+
+            # prepare test env
+            self.setup()
+
+            # wait for aodh alarms are updated in caches for event evaluator,
+            # sleep time should be larger than event_alarm_cache_ttl(default 60)
+            time.sleep(60)
+
+            # injecting host failure...
+            # NOTE (umar) add INTERFACE_NAME logic to host injection
+
+            self.fault.start(self.down_host)
+            time.sleep(10)
+
+            # verify the test results
+            # NOTE (umar) copy remote monitor.log file when monitor=collectd
+            self.check_host_status(self.down_host.name, 'down')
+
+            notification_time = calculate_notification_time()
+            if notification_time < 1 and notification_time > 0:
+                LOG.info('doctor test successfully, notification_time=%s' % notification_time)
+            else:
+                LOG.error('doctor test failed, notification_time=%s' % notification_time)
+                sys.exit(1)
+
+            if self.conf.profiler_type:
+                LOG.info('doctor test begin to run profile.......')
+                self.collect_logs()
+                self.run_profiler()
+        except Exception as e:
+            LOG.error('doctor test failed, Exception=%s' % e)
+            sys.exit(1)
+        finally:
+            self.cleanup()
+
+    def get_host_info_for_random_vm(self):
+        num = random.randint(0, self.conf.instance_count - 1)
+        vm_name = "%s%d" % (self.conf.instance_basename, num)
+
+        servers = \
+            {getattr(server, 'name'): server
+             for server in self.nova.servers.list()}
+        server = servers.get(vm_name)
+        if not server:
+            raise \
+                Exception('Can not find instance: vm_name(%s)' % vm_name)
+        host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname')
+        host_ip = self.installer.get_host_ip_from_hostname(host_name)
+
+        LOG.info('Get host info(name:%s, ip:%s) which vm(%s) launched at'
+                 % (host_name, host_ip, vm_name))
+        return Host(host_name, host_ip)
+
+    def check_host_status(self, hostname, state):
+        service = self.nova.services.list(host=hostname, binary='nova-compute')
+        host_state = service[0].__dict__.get('state')
+        assert host_state == state
+
+    def unset_forced_down_hosts(self):
+        if self.down_host:
+            self.nova.services.force_down(self.down_host.name, 'nova-compute', False)
+            time.sleep(2)
+            self.check_host_status(self.down_host.name, 'up')
+
+    def collect_logs(self):
+        self.fault.get_disable_network_log()
+
+    def run_profiler(self):
+
+        log_file = '{0}/{1}'.format(sys.path[0], 'disable_network.log')
+        reg = '(?<=doctor set link down at )\d+.\d+'
+        linkdown = float(match_rep_in_file(reg, log_file).group(0))
+
+        log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log')
+        reg = '(.* doctor mark vm.* error at )(\d+.\d+)'
+        vmdown = float(match_rep_in_file(reg, log_file).group(2))
+
+        reg = '(?<=doctor mark host.* down at )\d+.\d+'
+        hostdown = float(match_rep_in_file(reg, log_file).group(2))
+
+        reg = '(?<=doctor monitor detected at )\d+.\d+'
+        detected = float(match_rep_in_file(reg, log_file).group(0))
+
+        reg = '(?<=doctor consumer notified at )\d+.\d+'
+        notified = float(match_rep_in_file(reg, log_file).group(0))
+
+        # TODO(yujunz) check the actual delay to verify time sync status
+        # expected ~1s delay from $trigger to $linkdown
+        relative_start = linkdown
+        os.environ['DOCTOR_PROFILER_T00'] = str(int((linkdown - relative_start)*1000))
+        os.environ['DOCTOR_PROFILER_T01'] = str(int((detected - relative_start) * 1000))
+        os.environ['DOCTOR_PROFILER_T03'] = str(int((vmdown - relative_start) * 1000))
+        os.environ['DOCTOR_PROFILER_T04'] = str(int((hostdown - relative_start) * 1000))
+        os.environ['DOCTOR_PROFILER_T09'] = str(int((notified - relative_start) * 1000))
+
+        profiler_main(log=LOG)
+
+    def cleanup(self):
+        self.unset_forced_down_hosts()
+        self.inspector.stop()
+        self.monitor.stop()
+        self.consumer.stop()
+        self.installer.cleanup()
+        self.alarm.delete()
+        self.instance.delete()
+        self.network.delete()
+        self.image.delete()
+        self.fault.cleanup()
+        self.user.delete()
+
+
+def main():
+    """doctor main"""
+    test_dir = os.path.split(os.path.realpath(__file__))[0]
+    doctor_root_dir = os.path.dirname(test_dir)
+
+    config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/')
+    config_files = [join(config_file_dir, f) for f in os.listdir(config_file_dir)
+                    if isfile(join(config_file_dir, f))]
+
+    conf = config.prepare_conf(args=sys.argv[1:],
+                               config_files=config_files)
+
+    doctor = DoctorTest(conf)
+    doctor.run()
diff --git a/doctor_tests/monitor/__init__.py b/doctor_tests/monitor/__init__.py
new file mode 100644
index 00000000..7e30c9f8
--- /dev/null
+++ b/doctor_tests/monitor/__init__.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+    cfg.StrOpt('type',
+               default='sample',
+               choices=['sample', 'collectd'],
+               help='the type of doctor monitor component',
+               required=True),
+]
+
+
+_monitor_name_class_mapping = {
+    'sample': 'doctor_tests.monitor.sample.SampleMonitor',
+    'collectd': 'doctor_tests.monitor.collectd.CollectdMonitor'
+}
+
+def get_monitor(conf, inspector_url, log):
+    monitor_class = _monitor_name_class_mapping.get(conf.monitor.type)
+    return importutils.import_object(monitor_class, conf,
+                                     inspector_url, log)
diff --git a/doctor_tests/monitor/base.py b/doctor_tests/monitor/base.py
new file mode 100644
index 00000000..119c8a1c
--- /dev/null
+++ b/doctor_tests/monitor/base.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseMonitor(object):
+    """Monitor computer fault and report error to the inspector"""
+    def __init__(self, conf, inspector_url, log):
+        self.conf = conf
+        self.log = log
+        self.inspector_url = inspector_url
+
+    @abc.abstractmethod
+    def start(self, host):
+        pass
+
+    @abc.abstractmethod
+    def stop(self):
+        pass
diff --git a/doctor_tests/monitor/collectd.py b/doctor_tests/monitor/collectd.py
new file mode 100644
index 00000000..a22d7edc
--- /dev/null
+++ b/doctor_tests/monitor/collectd.py
@@ -0,0 +1,137 @@
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import socket
+import getpass
+import sys
+
+from doctor_tests.monitor.base import BaseMonitor
+
+
+class CollectdMonitor(BaseMonitor):
+    def __init__(self, conf, inspector_url, log):
+        super(CollectdMonitor, self).__init__(conf, inspector_url, log)
+        self.top_dir = os.path.dirname(sys.path[0])
+        tmp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        tmp_sock.connect(("8.8.8.8", 80))
+
+        ## control_ip is the IP of primary interface of control node i.e.
+        ## eth0, eno1. It is used by collectd monitor to communicate with
+        ## sample inspector.
+        ## TODO (umar) see if mgmt IP of control is a better option. Also
+        ## primary interface may not be the right option
+        self.control_ip = tmp_sock.getsockname()[0]
+        self.compute_user = getpass.getuser()
+        self.interface_name = os.environ.get('INTERFACE_NAME') or ''
+        self.inspector_type = os.environ.get('INSPECTOR_TYPE', 'sample')
+        self.auth_url = os.environ.get('OS_AUTH_URL')
+        self.username = os.environ.get('OS_USERNAME')
+        self.password = os.environ.get('OS_PASSWORD')
+        self.project_name = os.environ.get('OS_PROJECT_NAME')
+        self.user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default'
+        self.user_domain_id = os.environ.get('OS_USER_DOMAIN_ID')
+        self.project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default'
+        self.project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID')
+        self.ssh_opts_cpu = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+
+    def start(self, host):
+        self.log.info("Collectd monitor start.........")
+        self.compute_host = host.name
+        self.compute_ip = host.ip
+        f = open("%s/tests/collectd.conf" % self.top_dir, 'w')
+        collectd_conf_file = """ 
+Hostname %s
+FQDNLookup false
+Interval 1
+MaxReadInterval 2
+
+<LoadPlugin python>
+Globals true
+</LoadPlugin>
+LoadPlugin ovs_events
+LoadPlugin logfile
+
+<Plugin logfile>
+    File \"/var/log/collectd.log\"
+    Timestamp true
+    LogLevel \"info\"
+</Plugin>
+
+<Plugin python>
+    ModulePath \"/home/%s\"
+    LogTraces true
+    Interactive false
+    Import \"collectd_plugin\"
+    <Module \"collectd_plugin\">
+        control_ip \"%s\"
+        compute_ip \"%s\"
+        compute_host \"%s\"
+        compute_user \"%s\"
+        inspector_type \"%s\"
+        os_auth_url \"%s\"
+        os_username \"%s\"
+        os_password \"%s\"
+        os_project_name \"%s\"
+        os_user_domain_name \"%s\"
+        os_user_domain_id \"%s\"
+        os_project_domain_name \"%s\"
+        os_project_domain_id \"%s\"
+    </Module>
+</Plugin>
+
+<Plugin ovs_events>
+    Port 6640
+    Socket \"/var/run/openvswitch/db.sock\"
+    Interfaces \"@INTERFACE_NAME@\"
+    SendNotification true
+    DispatchValues false
+</Plugin>
+            """ % (self.compute_host, self.compute_user, self.control_ip, self.compute_ip, self.compute_host, self.compute_user,
+                   self.inspector_type, self.auth_url, self.username, self.password, self.project_name, self.user_domain_name,
+                   self.user_domain_id, self.project_domain_name, self.project_domain_id)
+        f.write(collectd_conf_file)
+        f.close()
+
+        os.system(" scp %s %s/tests/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+        self.log.info("after first scp")
+        ## @TODO (umar) Always assuming that the interface is assigned an IP if
+        ## interface name is not provided. See if there is a better approach
+        os.system(""" ssh %s %s@%s \"if [ -n \"%s\" ]; then
+            dev=%s
+        else
+            dev=\$(sudo ip a | awk '/ %s\//{print \$NF}')
+        fi
+        sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf
+        collectd_conf=/opt/collectd/etc/collectd.conf
+        if [ -e \$collectd_conf ]; then
+            sudo cp \$collectd_conf \${collectd_conf}-doctor-saved
+        else
+            sudo touch \${collectd_conf}-doctor-created
+        fi
+        sudo mv collectd.conf /opt/collectd/etc/collectd.conf\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip, self.interface_name, self.interface_name, self.compute_ip))
+        self.log.info("after first ssh")
+        os.system(" scp  %s %s/tests/lib/monitors/collectd/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+        self.log.info("after sec scp")
+        os.system(" ssh %s %s@%s \"sudo pkill collectd; sudo /opt/collectd/sbin/collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+        self.log.info("after sec ssh")
+
+    def stop(self):
+        os.system(" ssh %s %s@%s \"sudo pkill collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+
+    def cleanup(self):
+        os.system(""" ssh %s %s@%s \"
+            collectd_conf=/opt/collectd/etc/collectd.conf
+            if [ -e \"\${collectd_conf}-doctor-created\" ]; then
+                sudo rm \"\${collectd_conf}-doctor-created\"
+                sudo rm \$collectd_conf
+            elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then
+                sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf
+                sudo rm \"\${collectd_conf}-doctor-saved\"
+            fi\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+        os.remove("%s/tests/collectd.conf" % self.top_dir)
diff --git a/doctor_tests/monitor/sample.py b/doctor_tests/monitor/sample.py
new file mode 100644
index 00000000..7a463048
--- /dev/null
+++ b/doctor_tests/monitor/sample.py
@@ -0,0 +1,106 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from datetime import datetime
+import json
+import requests
+import socket
+from threading import Thread
+import time
+
+from doctor_tests.identity_auth import get_session
+from doctor_tests.monitor.base import BaseMonitor
+
+
+class SampleMonitor(BaseMonitor):
+    event_type = "compute.host.down"
+
+    def __init__(self, conf, inspector_url, log):
+        super(SampleMonitor, self).__init__(conf, inspector_url, log)
+        self.session = get_session()
+        self.pinger = None
+
+    def start(self, host):
+        self.log.info('sample monitor start......')
+        self.pinger = Pinger(host.name, host.ip, self, self.log)
+        self.pinger.start()
+
+    def stop(self):
+        self.log.info('sample monitor stop......')
+        if self.pinger is not None:
+            self.pinger.stop()
+            self.pinger.join()
+
+    def report_error(self, hostname):
+        self.log.info('sample monitor report error......')
+        data = [
+            {
+                'id': 'monitor_sample_id1',
+                'time': datetime.now().isoformat(),
+                'type': self.event_type,
+                'details': {
+                    'hostname': hostname,
+                    'status': 'down',
+                    'monitor': 'monitor_sample',
+                    'monitor_event_id': 'monitor_sample_event1'
+                },
+            },
+        ]
+
+        auth_token = self.session.get_token() if \
+                     self.conf.inspector.type != 'sample' else None
+        headers = {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            'X-Auth-Token': auth_token,
+        }
+
+        url = '%s%s' % (self.inspector_url, 'events') \
+            if self.inspector_url.endswith('/') else \
+            '%s%s' % (self.inspector_url, '/events')
+        requests.put(url, data=json.dumps(data), headers=headers)
+
+
+class Pinger(Thread):
+    interval = 0.1  # second
+    timeout = 0.1   # second
+    ICMP_ECHO_MESSAGE = bytes([0x08, 0x00, 0xf7, 0xff, 0x00, 0x00, 0x00, 0x00])
+
+    def __init__(self, host_name, host_ip, monitor, log):
+        Thread.__init__(self)
+        self.monitor = monitor
+        self.hostname = host_name
+        self.ip_addr = host_ip or socket.gethostbyname(self.hostname)
+        self.log = log
+        self._stopped = False
+
+    def run(self):
+        self.log.info("Starting Pinger host_name(%s), host_ip(%s)"
+                      % (self.hostname, self.ip_addr))
+
+        sock = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                             socket.IPPROTO_ICMP)
+        sock.settimeout(self.timeout)
+        while True:
+            if self._stopped:
+                return
+            try:
+                sock.sendto(self.ICMP_ECHO_MESSAGE, (self.ip_addr, 0))
+                sock.recv(4096)
+            except socket.timeout:
+                self.log.info("doctor monitor detected at %s" % time.time())
+                self.monitor.report_error(self.hostname)
+                self.log.info("ping timeout, quit monitoring...")
+                self._stopped = True
+                return
+            time.sleep(self.interval)
+
+    def stop(self):
+        self.log.info("Stopping Pinger host_name(%s), host_ip(%s)"
+                      % (self.hostname, self.ip_addr))
+        self._stopped = True
diff --git a/doctor_tests/network.py b/doctor_tests/network.py
new file mode 100644
index 00000000..ee153e66
--- /dev/null
+++ b/doctor_tests/network.py
@@ -0,0 +1,68 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import neutron_client
+
+
+OPTS = [
+    cfg.StrOpt('net_name',
+               default='doctor_net',
+               help='the name of test net',
+               required=True),
+    cfg.StrOpt('net_cidr',
+               default='192.168.168.0/24',
+               help='the cidr of test subnet',
+               required=True),
+]
+
+
+class Network(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+        self.auth = get_identity_auth(username=self.conf.doctor_user,
+                                      password=self.conf.doctor_passwd,
+                                      project=self.conf.doctor_project)
+        self.neutron = neutron_client(get_session(auth=self.auth))
+        self.net = None
+        self.subnet = None
+
+    def create(self):
+        self.log.info('network create start.......')
+        net_name = self.conf.net_name
+        networks = self.neutron.list_networks(name=net_name)['networks']
+        self.net = networks[0] if networks \
+            else self.neutron.create_network(
+            {'network': {'name': net_name}})['network']
+        self.log.info('network create end.......')
+
+        self.log.info('subnet create start.......')
+        subnets = self.neutron.list_subnets(network_id=self.net['id'])['subnets']
+        subnet_param = {'name': net_name, 'network_id': self.net['id'],
+                        'cidr': self.conf.net_cidr, 'ip_version': 4,
+                        'enable_dhcp': False}
+        self.subnet = subnets[0] if subnets \
+            else self.neutron.create_subnet(
+            {'subnet': subnet_param})['subnet']
+        self.log.info('subnet create end.......')
+
+    def delete(self):
+        self.log.info('subnet delete start.......')
+        if self.subnet:
+            self.neutron.delete_subnet(self.subnet['id'])
+        self.log.info('subnet delete end.......')
+
+        self.log.info('network delete start.......')
+        if self.net:
+            self.neutron.delete_network(self.net['id'])
+        self.log.info('network delete end.......')
diff --git a/doctor_tests/os_clients.py b/doctor_tests/os_clients.py
new file mode 100644
index 00000000..44fa3aad
--- /dev/null
+++ b/doctor_tests/os_clients.py
@@ -0,0 +1,50 @@
+##############################################################################

+# Copyright (c) 2017 ZTE Corporation and others.

+#

+# All rights reserved. This program and the accompanying materials

+# are made available under the terms of the Apache License, Version 2.0

+# which accompanies this distribution, and is available at

+# http://www.apache.org/licenses/LICENSE-2.0

+##############################################################################

+from oslo_config import cfg

+

+import aodhclient.client as aodhclient

+from congressclient.v1 import client as congressclient

+import glanceclient.client as glanceclient

+from keystoneclient.v2_0 import client as ks_client

+from neutronclient.v2_0 import client as neutronclient

+import novaclient.client as novaclient

+

+

+OPTS = [

+    cfg.StrOpt('glance_version', default='2', help='glance version'),

+    cfg.StrOpt('nova_version', default='2.34', help='Nova version'),

+    cfg.StrOpt('aodh_version', default='2', help='aodh version'),

+]

+

+

+def glance_client(version, session):

+    return glanceclient.Client(version=version,

+                               session=session)

+

+

+def keystone_client(session):

+    return ks_client.Client(session=session)

+

+

+def nova_client(version, session):

+    return novaclient.Client(version=version,

+                             session=session)

+

+

+def neutron_client(session):

+    return neutronclient.Client(session=session)

+

+

+def aodh_client(version, session):

+    return aodhclient.Client(version, session=session)

+

+

+def congress_client(session):

+    return congressclient.Client(session=session,

+                                 service_type='policy')

diff --git a/doctor_tests/profiler_poc.py b/doctor_tests/profiler_poc.py
new file mode 100644
index 00000000..ea36eaeb
--- /dev/null
+++ b/doctor_tests/profiler_poc.py
@@ -0,0 +1,100 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+"""
+PoC of performance profiler for OPNFV doctor project
+
+Usage:
+
+Export environment variables to set timestamp at each checkpoint in millisecond.
+Valid check points are: DOCTOR_PROFILER_T{00-09}
+
+See also: https://goo.gl/98Osig
+"""
+
+import json
+import os
+
+from oslo_config import cfg
+
+
+OPTS = [
+    cfg.StrOpt('profiler_type',
+               default=os.environ.get('PROFILER_TYPE', 'poc'),
+               help='the type of installer'),
+]
+
+
+OUTPUT = 'doctor_profiling_output'
+PREFIX = 'DOCTOR_PROFILER'
+TOTAL_CHECK_POINTS = 10
+MODULE_CHECK_POINTS = ['T00', 'T01', 'T04', 'T05', 'T06', 'T09']
+TAG_FORMAT = "{:<5}"
+# Inspired by https://github.com/reorx/httpstat
+TEMPLATE = """
+Total time cost: {total}(ms)
+==============================================================================>
+       |Monitor|Inspector           |Controller|Notifier|Evaluator           |
+       |{M00}  |{M01}               |{M02}     |{M03}   |{M04}               |
+       |       |      |      |      |          |        |      |      |      |
+link down:{T00}|      |      |      |          |        |      |      |      |
+     raw failure:{T01}|      |      |          |        |      |      |      |
+         found affected:{T02}|      |          |        |      |      |      |
+                  set VM error:{T03}|          |        |      |      |      |
+                         marked host down:{T04}|        |      |      |      |
+                               notified VM error:{T05}  |      |      |      |
+                                        transformed event:{T06}|      |      |
+                                                 evaluated event:{T07}|      |
+                                                            fired alarm:{T08}|
+                                                                received alarm:{T09}
+"""
+
+
+def main(log=None):
+    check_points = ["T{:02d}".format(i) for i in range(TOTAL_CHECK_POINTS)]
+    module_map = {"M{:02d}".format(i):
+                  (MODULE_CHECK_POINTS[i], MODULE_CHECK_POINTS[i + 1])
+                  for i in range(len(MODULE_CHECK_POINTS) - 1)}
+
+    # check point tags
+    elapsed_ms = {cp: os.getenv("{}_{}".format(PREFIX, cp))
+                  for cp in check_points}
+
+    def format_tag(tag):
+        return TAG_FORMAT.format(tag or '?')
+
+    tags = {cp: format_tag(ms) for cp, ms in elapsed_ms.items()}
+
+    def time_cost(cp):
+        if elapsed_ms[cp[0]] and elapsed_ms[cp[1]]:
+            return int(elapsed_ms[cp[1]]) - int(elapsed_ms[cp[0]])
+        else:
+            return None
+
+    # module time cost tags
+    modules_cost_ms = {module: time_cost(cp)
+                       for module, cp in module_map.items()}
+
+    tags.update({module: format_tag(cost)
+                 for module, cost in modules_cost_ms.items()})
+
+    tags.update({'total': time_cost((check_points[0], check_points[-1]))})
+
+    profile = TEMPLATE.format(**tags)
+
+    logfile = open('{}.json'.format(OUTPUT), 'w')
+    logfile.write(json.dumps(tags))
+
+    print(profile)
+    if log:
+        log.info('%s' % profile)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/doctor_tests/scenario/__init__.py b/doctor_tests/scenario/__init__.py
new file mode 100644
index 00000000..48893ae6
--- /dev/null
+++ b/doctor_tests/scenario/__init__.py
@@ -0,0 +1,8 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
diff --git a/doctor_tests/scenario/common.py b/doctor_tests/scenario/common.py
new file mode 100644
index 00000000..a5cbe483
--- /dev/null
+++ b/doctor_tests/scenario/common.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import sys
+
+from doctor_tests.common.utils import match_rep_in_file
+
+
+def calculate_notification_time():
+    log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log')
+
+    reg = '(?<=doctor monitor detected at )\d+.\d+'
+    result = match_rep_in_file(reg, log_file)
+    if not result:
+        raise Exception('Can not match detected time')
+    detected = result.group(0)
+
+    reg = '(?<=doctor consumer notified at )\d+.\d+'
+    result = match_rep_in_file(reg, log_file)
+    if not result:
+        raise Exception('Can not match notified time')
+    notified = result.group(0)
+
+    return float(notified) - float(detected)
diff --git a/doctor_tests/scenario/network_failure.py b/doctor_tests/scenario/network_failure.py
new file mode 100644
index 00000000..b94a622d
--- /dev/null
+++ b/doctor_tests/scenario/network_failure.py
@@ -0,0 +1,71 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import nova_client
+from doctor_tests.common.utils import SSHClient
+
+LINK_DOWN_SCRIPT = """
+#!/bin/bash -x
+dev=$(sudo ip a | awk '/ {compute_ip}\//{{print $NF}}')
+sleep 1
+sudo ip link set $dev down
+echo "doctor set link down at" $(date "+%s.%N")
+sleep 10
+sudo ip link set $dev up
+sleep 1
+"""
+
+
+class NetworkFault(object):
+
+    def __init__(self, conf, installer, log):
+        self.conf = conf
+        self.log = log
+        self.installer = installer
+        self.nova = nova_client(self.conf.nova_version, get_session())
+        self.host = None
+        self.GetLog = False
+
+    def start(self, host):
+        self.log.info('fault inject start......')
+        self._set_link_down(host.ip)
+        self.host = host
+        self.log.info('fault inject end......')
+
+    def cleanup(self):
+        self.log.info('fault inject cleanup......')
+        self.get_disable_network_log()
+
+    def get_disable_network_log(self):
+        if self.GetLog:
+            self.log.info('Already get the disable_netork.log from down_host......')
+            return
+        if self.host is not None:
+            client = SSHClient(self.host.ip,
+                               self.installer.node_user_name,
+                               key_filename=self.installer.get_ssh_key_from_installer(),
+                               look_for_keys=True,
+                               log=self.log)
+            client.scp('disable_network.log', './disable_network.log', method='get')
+            self.log.info('Get the disable_netork.log from down_host(host_name:%s, host_ip:%s)'
+                          % (self.host.name, self.host.ip))
+        self.GetLog = True
+
+    def _set_link_down(self, compute_ip):
+        file_name = './disable_network.sh'
+        with open(file_name, 'w') as file:
+            file.write(LINK_DOWN_SCRIPT.format(compute_ip=compute_ip))
+        client = SSHClient(compute_ip,
+                           self.installer.node_user_name,
+                           key_filename=self.installer.get_ssh_key_from_installer(),
+                           look_for_keys=True,
+                           log=self.log)
+        client.scp('./disable_network.sh', 'disable_network.sh')
+        command = 'bash disable_network.sh > disable_network.log 2>&1 &'
+        client.ssh(command)
diff --git a/doctor_tests/user.py b/doctor_tests/user.py
new file mode 100644
index 00000000..33f995e7
--- /dev/null
+++ b/doctor_tests/user.py
@@ -0,0 +1,163 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import keystone_client
+from doctor_tests.os_clients import nova_client
+
+
+OPTS = [
+    cfg.StrOpt('doctor_user',
+               default='doctor',
+               help='the name of test user',
+               required=True),
+    cfg.StrOpt('doctor_passwd',
+               default='doctor',
+               help='the password of test user',
+               required=True),
+    cfg.StrOpt('doctor_project',
+               default='doctor',
+               help='the name of test project',
+               required=True),
+    cfg.StrOpt('doctor_role',
+               default='_member_',
+               help='the role of test user',
+               required=True),
+    cfg.IntOpt('quota_instances',
+               default=os.environ.get('VM_COUNT', 1),
+               help='the quota of instances in test user',
+               required=True),
+    cfg.IntOpt('quota_cores',
+               default=os.environ.get('VM_COUNT', 1),
+               help='the quota of cores in test user',
+               required=True),
+]
+
+
+class User(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+        self.keystone = \
+            keystone_client(get_session())
+        self.nova = \
+            nova_client(conf.nova_version, get_session())
+        self.users = {}
+        self.projects = {}
+        self.roles = {}
+        self.roles_for_user = {}
+        self.roles_for_admin = {}
+
+    def create(self):
+        """create test user, project and etc"""
+        self.log.info('user create start......')
+
+        self._create_project()
+        self._create_user()
+        self._create_role()
+        self._add_user_role_in_project(is_admin=False)
+        self._add_user_role_in_project(is_admin=True)
+
+        self.log.info('user create end......')
+
+    def _create_project(self):
+        """create test project"""
+        self.projects = {project.name: project
+                    for project in self.keystone.tenants.list()}
+        if self.conf.doctor_project not in self.projects:
+            test_project = \
+                self.keystone.tenants.create(self.conf.doctor_project)
+            self.projects[test_project.name] = test_project
+
+    def _create_user(self):
+        """create test user"""
+        project = self.projects.get(self.conf.doctor_project)
+        self.users = {user.name: user for user in self.keystone.users.list()}
+        if self.conf.doctor_user not in self.users:
+            test_user = self.keystone.users.create(
+                self.conf.doctor_user,
+                password=self.conf.doctor_passwd,
+                tenant_id=project.id)
+            self.users[test_user.name] = test_user
+
+    def _create_role(self):
+        """create test role"""
+        self.roles = {role.name: role for role in self.keystone.roles.list()}
+        if self.conf.doctor_role not in self.roles:
+            test_role = self.keystone.roles.create(self.conf.doctor_role)
+            self.roles[test_role.name] = test_role
+
+    def _add_user_role_in_project(self, is_admin=False):
+        """add test user with test role in test project"""
+        project = self.projects.get(self.conf.doctor_project)
+
+        user_name = 'admin' if is_admin else self.conf.doctor_user
+        user = self.users.get(user_name)
+
+        role_name = 'admin' if is_admin else self.conf.doctor_role
+        role = self.roles.get(role_name)
+
+        roles_for_user = self.roles_for_admin \
+            if is_admin else self.roles_for_user
+
+        roles_for_user = \
+            {role.name: role for role in
+             self.keystone.roles.roles_for_user(user, tenant=project)}
+        if role_name not in roles_for_user:
+            self.keystone.roles.add_user_role(user, role, tenant=project)
+            roles_for_user[role_name] = role
+
+    def delete(self):
+        """delete the test user, project and role"""
+        self.log.info('user delete start......')
+
+        project = self.projects.get(self.conf.doctor_project)
+        user = self.users.get(self.conf.doctor_user)
+        role = self.roles.get(self.conf.doctor_role)
+
+        if project:
+            if 'admin' in self.roles_for_admin:
+                self.keystone.roles.remove_user_role(
+                    self.users['admin'],
+                    self.roles['admin'],
+                    tenant=project)
+
+            if user:
+                if role and self.conf.doctor_role in self.roles_for_user:
+                    self.keystone.roles.remove_user_role(
+                        user, role, tenant=project)
+                    self.keystone.roles.delete(role)
+                self.keystone.users.delete(user)
+
+            self.keystone.tenants.delete(project)
+        self.log.info('user delete end......')
+
+    def update_quota(self):
+        self.log.info('user quota update start......')
+        project = self.projects.get(self.conf.doctor_project)
+        user = self.users.get(self.conf.doctor_user)
+
+        if project and user:
+            self.quota = self.nova.quotas.get(project.id,
+                                              user_id=user.id)
+            if self.conf.quota_instances > self.quota.instances:
+                self.nova.quotas.update(project.id,
+                                        instances=self.conf.quota_instances,
+                                        user_id=user.id)
+            if self.conf.quota_cores > self.quota.cores:
+                self.nova.quotas.update(project.id,
+                                        cores=self.conf.quota_cores,
+                                        user_id=user.id)
+            self.log.info('user quota update end......')
+        else:
+            raise Exception('No project or role for update quota')