summaryrefslogtreecommitdiffstats
path: root/doctor_tests
diff options
context:
space:
mode:
Diffstat (limited to 'doctor_tests')
-rw-r--r--doctor_tests/alarm.py93
-rw-r--r--doctor_tests/common/__init__.py8
-rw-r--r--doctor_tests/common/constants.py12
-rw-r--r--doctor_tests/common/utils.py105
-rw-r--r--doctor_tests/config.py54
-rw-r--r--doctor_tests/consumer/__init__.py37
-rw-r--r--doctor_tests/consumer/base.py26
-rw-r--r--doctor_tests/consumer/sample.py71
-rw-r--r--doctor_tests/identity_auth.py43
-rw-r--r--doctor_tests/image.py74
-rw-r--r--doctor_tests/inspector/__init__.py40
-rw-r--r--doctor_tests/inspector/base.py30
-rw-r--r--doctor_tests/inspector/congress.py94
-rw-r--r--doctor_tests/inspector/sample.py169
-rw-r--r--doctor_tests/installer/__init__.py38
-rw-r--r--doctor_tests/installer/apex.py126
-rw-r--r--doctor_tests/installer/base.py36
-rw-r--r--doctor_tests/installer/common/congress.py47
-rw-r--r--doctor_tests/installer/common/restore_ceilometer.py27
-rw-r--r--doctor_tests/installer/common/set_ceilometer.py44
-rw-r--r--doctor_tests/installer/local.py109
-rw-r--r--doctor_tests/instance.py114
-rw-r--r--doctor_tests/logger.py46
-rw-r--r--doctor_tests/main.py215
-rw-r--r--doctor_tests/monitor/__init__.py29
-rw-r--r--doctor_tests/monitor/base.py27
-rw-r--r--doctor_tests/monitor/collectd.py137
-rw-r--r--doctor_tests/monitor/sample.py106
-rw-r--r--doctor_tests/network.py68
-rw-r--r--doctor_tests/os_clients.py50
-rw-r--r--doctor_tests/profiler_poc.py100
-rw-r--r--doctor_tests/scenario/__init__.py8
-rw-r--r--doctor_tests/scenario/common.py29
-rw-r--r--doctor_tests/scenario/network_failure.py71
-rw-r--r--doctor_tests/user.py163
35 files changed, 2446 insertions, 0 deletions
diff --git a/doctor_tests/alarm.py b/doctor_tests/alarm.py
new file mode 100644
index 00000000..3b1aaf3f
--- /dev/null
+++ b/doctor_tests/alarm.py
@@ -0,0 +1,93 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import aodh_client
+from doctor_tests.os_clients import nova_client
+
+OPTS = [
+ cfg.StrOpt('alarm_basename',
+ default='doctor_alarm',
+ help='the base name of alarm',
+ required=True),
+]
+
+
+class Alarm(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(project=self.conf.doctor_project)
+ self.aodh = \
+ aodh_client(conf.aodh_version,
+ get_session(auth=self.auth))
+ self.nova = \
+ nova_client(conf.nova_version,
+ get_session(auth=self.auth))
+ self._init_alarm_name()
+
+ def _init_alarm_name(self):
+ self.alarm_names = []
+ for i in range(0, self.conf.instance_count):
+ alarm_name = '%s%d' % (self.conf.alarm_basename, i)
+ self.alarm_names.append(alarm_name)
+
+ def create(self):
+ self.log.info('alarm create start......')
+
+ alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()}
+ servers = \
+ {getattr(server, 'name'): server
+ for server in self.nova.servers.list()}
+
+ for i in range(0, self.conf.instance_count):
+ alarm_name = self.alarm_names[i]
+ if alarm_name in alarms:
+ continue;
+ vm_name = '%s%d' % (self.conf.instance_basename, i)
+ vm_id = getattr(servers[vm_name], 'id')
+ alarm_request = dict(
+ name=alarm_name,
+ description=u'VM failure',
+ enabled=True,
+ alarm_actions=[u'http://%s:%d/failure'
+ % (self.conf.consumer.ip,
+ self.conf.consumer.port)],
+ repeat_actions=False,
+ severity=u'moderate',
+ type=u'event',
+ event_rule=dict(
+ event_type=u'compute.instance.update',
+ query=[
+ dict(field=u'traits.instance_id',
+ type='',
+ op=u'eq',
+ value=vm_id),
+ dict(field=u'traits.state',
+ type='',
+ op=u'eq',
+ value=u'error')]))
+ self.aodh.alarm.create(alarm_request)
+
+ self.log.info('alarm create end......')
+
+ def delete(self):
+ self.log.info('alarm delete start.......')
+
+ alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()}
+ for alarm_name in self.alarm_names:
+ if alarm_name in alarms:
+ self.aodh.alarm.delete(alarms[alarm_name]['alarm_id'])
+
+ del self.alarm_names[:]
+
+ self.log.info('alarm delete end.......')
diff --git a/doctor_tests/common/__init__.py b/doctor_tests/common/__init__.py
new file mode 100644
index 00000000..e68a3070
--- /dev/null
+++ b/doctor_tests/common/__init__.py
@@ -0,0 +1,8 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+############################################################################## \ No newline at end of file
diff --git a/doctor_tests/common/constants.py b/doctor_tests/common/constants.py
new file mode 100644
index 00000000..72d037af
--- /dev/null
+++ b/doctor_tests/common/constants.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from collections import namedtuple
+
+
+Host = namedtuple('Host', ['name', 'ip'])
diff --git a/doctor_tests/common/utils.py b/doctor_tests/common/utils.py
new file mode 100644
index 00000000..2e823acb
--- /dev/null
+++ b/doctor_tests/common/utils.py
@@ -0,0 +1,105 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import json
+import os
+import paramiko
+import re
+
+
+def load_json_file(full_path):
+ """Loads JSON from file
+ :param target_filename:
+ :return:
+ """
+ if not os.path.isfile(full_path):
+ raise Exception('File(%s) does not exist' % full_path)
+
+ with open(full_path, 'r') as file:
+ return json.load(file)
+
+
+def write_json_file(full_path, data):
+ """write JSON from file
+ :param target_filename:
+ :return:
+ """
+
+ with open(full_path, 'w+') as file:
+ file.write(json.dumps(data))
+
+
+def match_rep_in_file(regex, full_path):
+ if not os.path.isfile(full_path):
+ raise Exception('File(%s) does not exist' % full_path)
+
+ with open(full_path, 'r') as file:
+ for line in file:
+ result = re.search(regex, line)
+ if result:
+ return result
+
+ return None
+
+
+class SSHClient(object):
+ def __init__(self, ip, username, password=None, pkey=None,
+ key_filename=None, log=None, look_for_keys=False,
+ allow_agent=False):
+ self.client = paramiko.SSHClient()
+ self.client.load_system_host_keys()
+ self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ self.client.connect(ip, username=username, password=password,
+ pkey=pkey, key_filename=key_filename,
+ look_for_keys=look_for_keys,
+ allow_agent=allow_agent)
+ self.log = log
+
+ def __del__(self):
+ self.client.close()
+
+ def ssh(self, command):
+ if self.log:
+ self.log.info("Executing: %s" % command)
+ stdin, stdout, stderr = self.client.exec_command(command)
+ ret = stdout.channel.recv_exit_status()
+ output = list()
+ for line in stdout.read().splitlines():
+ output.append(line.decode('utf-8'))
+ if ret:
+ if self.log:
+ self.log.info("*** FAILED to run command %s (%s)" % (command, ret))
+ raise Exception(
+ "Unable to run \ncommand: %s\nret: %s"
+ % (command, ret))
+ if self.log:
+ self.log.info("*** SUCCESSFULLY run command %s" % command)
+ return ret, output
+
+ def scp(self, source, dest, method='put'):
+ if self.log:
+ self.log.info("Copy %s -> %s" % (source, dest))
+ ftp = self.client.open_sftp()
+ if method == 'put':
+ ftp.put(source, dest)
+ elif method == 'get':
+ ftp.get(source, dest)
+ ftp.close()
+
+
+def run_async(func):
+ from threading import Thread
+ from functools import wraps
+
+ @wraps(func)
+ def async_func(*args, **kwargs):
+ thread = Thread(target=func, args=args, kwargs=kwargs)
+ thread.start()
+ return thread
+
+ return async_func
diff --git a/doctor_tests/config.py b/doctor_tests/config.py
new file mode 100644
index 00000000..273e84d5
--- /dev/null
+++ b/doctor_tests/config.py
@@ -0,0 +1,54 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import itertools
+
+from oslo_config import cfg
+
+from doctor_tests import alarm
+from doctor_tests import consumer
+from doctor_tests import image
+from doctor_tests import instance
+from doctor_tests import installer
+from doctor_tests import network
+from doctor_tests import inspector
+from doctor_tests import monitor
+from doctor_tests import os_clients
+from doctor_tests import profiler_poc
+from doctor_tests import user
+
+
+def list_opts():
+ return [
+ ('installer', installer.OPTS),
+ ('monitor', monitor.OPTS),
+ ('inspector', inspector.OPTS),
+ ('consumer', consumer.OPTS),
+ ('DEFAULT', itertools.chain(
+ os_clients.OPTS,
+ image.OPTS,
+ user.OPTS,
+ network.OPTS,
+ instance.OPTS,
+ alarm.OPTS,
+ profiler_poc.OPTS))
+ ]
+
+
+def prepare_conf(args=None, conf=None, config_files=None):
+ if conf is None:
+ conf = cfg.ConfigOpts()
+
+ for group, options in list_opts():
+ conf.register_opts(list(options),
+ group=None if group == 'DEFAULT' else group)
+
+ conf(args, project='doctor', validate_default_values=True,
+ default_config_files=config_files)
+
+ return conf
diff --git a/doctor_tests/consumer/__init__.py b/doctor_tests/consumer/__init__.py
new file mode 100644
index 00000000..2c66a547
--- /dev/null
+++ b/doctor_tests/consumer/__init__.py
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+ cfg.StrOpt('type',
+ default='sample',
+ choices=['sample'],
+ help='the component of doctor consumer',
+ required=True),
+ cfg.StrOpt('ip',
+ default='127.0.0.1',
+ help='the ip of consumer',
+ required=True),
+ cfg.IntOpt('port',
+ default='12346',
+ help='the port of doctor consumer',
+ required=True),
+]
+
+
+_consumer_name_class_mapping = {
+ 'sample': 'doctor_tests.consumer.sample.SampleConsumer'
+}
+
+
+def get_consumer(conf, log):
+ consumer_class = _consumer_name_class_mapping.get(conf.consumer.type)
+ return importutils.import_object(consumer_class, conf, log)
diff --git a/doctor_tests/consumer/base.py b/doctor_tests/consumer/base.py
new file mode 100644
index 00000000..35170748
--- /dev/null
+++ b/doctor_tests/consumer/base.py
@@ -0,0 +1,26 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseConsumer(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass \ No newline at end of file
diff --git a/doctor_tests/consumer/sample.py b/doctor_tests/consumer/sample.py
new file mode 100644
index 00000000..d76a764b
--- /dev/null
+++ b/doctor_tests/consumer/sample.py
@@ -0,0 +1,71 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from doctor_tests.consumer.base import BaseConsumer
+
+
+class SampleConsumer(BaseConsumer):
+
+ def __init__(self, conf, log):
+ super(SampleConsumer, self).__init__(conf, log)
+ self.app = None
+
+ def start(self):
+ self.log.info('sample consumer start......')
+ self.app = ConsumerApp(self.conf.consumer.port, self, self.log)
+ self.app.start()
+
+ def stop(self):
+ self.log.info('sample consumer stop......')
+ if not self.app:
+ return
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ }
+ url = 'http://%s:%d/shutdown'\
+ % (self.conf.consumer.ip,
+ self.conf.consumer.port)
+ requests.post(url, data='', headers=headers)
+
+
+class ConsumerApp(Thread):
+
+ def __init__(self, port, consumer, log):
+ Thread.__init__(self)
+ self.port = port
+ self.consumer = consumer
+ self.log = log
+
+ def run(self):
+ app = Flask('consumer')
+
+ @app.route('/failure', methods=['POST'])
+ def event_posted():
+ self.log.info('doctor consumer notified at %s' % time.time())
+ self.log.info('sample consumer received data = %s' % request.data)
+ data = json.loads(request.data.decode('utf8'))
+ return 'OK'
+
+ @app.route('/shutdown', methods=['POST'])
+ def shutdown():
+ self.log.info('shutdown consumer app server at %s' % time.time())
+ func = request.environ.get('werkzeug.server.shutdown')
+ if func is None:
+ raise RuntimeError('Not running with the Werkzeug Server')
+ func()
+ return 'consumer app shutting down...'
+
+ app.run(host="0.0.0.0", port=self.port)
diff --git a/doctor_tests/identity_auth.py b/doctor_tests/identity_auth.py
new file mode 100644
index 00000000..2586720c
--- /dev/null
+++ b/doctor_tests/identity_auth.py
@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from keystoneauth1 import loading
+from keystoneauth1 import session
+
+
+def get_identity_auth(username=None, password=None, project=None):
+ auth_url = os.environ['OS_AUTH_URL']
+ username = username or os.environ['OS_USERNAME']
+ password = password or os.environ['OS_PASSWORD']
+ user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default'
+ user_domain_id = os.environ.get('OS_USER_DOMAIN_ID') or 'default'
+ project_name = project or os.environ.get('OS_PROJECT_NAME') \
+ or os.environ.get('OS_TENANT_NAME')
+ project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default'
+ project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID') or 'default'
+
+ loader = loading.get_plugin_loader('password')
+ return loader.load_from_options(
+ auth_url=auth_url,
+ username=username,
+ password=password,
+ user_domain_name=user_domain_name,
+ user_domain_id=user_domain_id,
+ project_name=project_name,
+ tenant_name=project_name,
+ project_domain_name=project_domain_name,
+ project_domain_id=project_domain_id)
+
+
+def get_session(auth=None):
+ """Get a user credentials auth session."""
+ if auth is None:
+ auth = get_identity_auth()
+ return session.Session(auth=auth)
diff --git a/doctor_tests/image.py b/doctor_tests/image.py
new file mode 100644
index 00000000..2e313e12
--- /dev/null
+++ b/doctor_tests/image.py
@@ -0,0 +1,74 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import urllib.request
+
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import glance_client
+
+OPTS = [
+ cfg.StrOpt('image_name',
+ default=os.environ.get('IMAGE_NAME', 'cirros'),
+ help='the name of test image',
+ required=True),
+ cfg.StrOpt('image_format',
+ default='qcow2',
+ help='the format of test image',
+ required=True),
+ cfg.StrOpt('image_filename',
+ default='cirros.img',
+ help='the name of image file',
+ required=True),
+ cfg.StrOpt('image_download_url',
+ default='https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img',
+ help='the url where to get the image',
+ required=True),
+]
+
+
+class Image(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.glance = \
+ glance_client(conf.glance_version, get_session())
+ self.use_existing_image = False
+ self.image = None
+
+ def create(self):
+ self.log.info('image create start......')
+
+ images = {image.name: image for image in self.glance.images.list()}
+ if self.conf.image_name not in images:
+ if not os.path.exists(self.conf.image_filename):
+ resp = urllib.request.urlopen(self.conf.image_download_url)
+ with open(self.conf.image_filename, "wb") as file:
+ file.write(resp.read())
+ self.image = self.glance.images.create(name=self.conf.image_name,
+ disk_format=self.conf.image_format,
+ container_format="bare",
+ visibility="public")
+ self.glance.images.upload(self.image['id'],
+ open(self.conf.image_filename, 'rb'))
+ else:
+ self.use_existing_image = True
+ self.image = images[self.conf.image_name]
+
+ self.log.info('image create end......')
+
+ def delete(self):
+ self.log.info('image delete start.......')
+
+ if not self.use_existing_image and self.image:
+ self.glance.images.delete(self.image['id'])
+
+ self.log.info('image delete end.......')
diff --git a/doctor_tests/inspector/__init__.py b/doctor_tests/inspector/__init__.py
new file mode 100644
index 00000000..3be79e57
--- /dev/null
+++ b/doctor_tests/inspector/__init__.py
@@ -0,0 +1,40 @@
+#############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+ cfg.StrOpt('type',
+ default=os.environ.get('INSPECTOR_TYPE', 'sample'),
+ choices=['sample', 'congress', 'vitrage'],
+ help='the component of doctor inspector',
+ required=True),
+ cfg.StrOpt('ip',
+ default='127.0.0.1',
+ help='the host ip of inspector',
+ required=False),
+ cfg.StrOpt('port',
+ default='12345',
+ help='the port of default for inspector',
+ required=False),
+]
+
+
+_inspector_name_class_mapping = {
+ 'sample': 'doctor_tests.inspector.sample.SampleInspector',
+ 'congress': 'doctor_tests.inspector.congress.CongressInspector',
+}
+
+
+def get_inspector(conf, log):
+ inspector_class = _inspector_name_class_mapping[conf.inspector.type]
+ return importutils.import_object(inspector_class, conf, log)
diff --git a/doctor_tests/inspector/base.py b/doctor_tests/inspector/base.py
new file mode 100644
index 00000000..854f0695
--- /dev/null
+++ b/doctor_tests/inspector/base.py
@@ -0,0 +1,30 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInspector(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractmethod
+ def get_inspector_url(self):
+ pass
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass \ No newline at end of file
diff --git a/doctor_tests/inspector/congress.py b/doctor_tests/inspector/congress.py
new file mode 100644
index 00000000..c89a41bd
--- /dev/null
+++ b/doctor_tests/inspector/congress.py
@@ -0,0 +1,94 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import congress_client
+
+from doctor_tests.inspector.base import BaseInspector
+
+
+class CongressInspector(BaseInspector):
+ nova_api_min_version = '2.11'
+ doctor_driver = 'doctor'
+ doctor_datasource = 'doctor'
+ policy = 'classification'
+ rules = {
+ 'host_down':
+ 'host_down(host) :- doctor:events(hostname=host, type="compute.host.down", status="down")',
+ 'active_instance_in_host':
+ 'active_instance_in_host(vmid, host) :- nova:servers(id=vmid, host_name=host, status="ACTIVE")',
+ 'host_force_down':
+ 'execute[nova:services.force_down(host, "nova-compute", "True")] :- host_down(host)',
+ 'error_vm_states':
+ 'execute[nova:servers.reset_state(vmid, "error")] :- host_down(host), active_instance_in_host(vmid, host)'
+ }
+
+ def __init__(self, conf, log):
+ super(CongressInspector, self).__init__(conf, log)
+ self.auth = get_identity_auth()
+ self.congress = congress_client(get_session(auth=self.auth))
+ self._init_driver_and_ds()
+ self.inspector_url = self.get_inspector_url()
+
+ def _init_driver_and_ds(self):
+ datasources = \
+ {ds['name']: ds for ds in self.congress.list_datasources()['results']}
+
+ # check nova_api version
+ nova_api_version = datasources['nova']['config'].get('api_version')
+ if nova_api_version and nova_api_version < self.nova_api_min_version:
+ raise Exception('Congress Nova datasource API version < nova_api_min_version(%s)'
+ % self.nova_api_min_version)
+
+ # create doctor datasource if it's not exist
+ if self.doctor_datasource not in datasources:
+ self.congress.create_datasource(
+ body={'driver': self.doctor_driver,
+ 'name': self.doctor_datasource})
+
+ # check whether doctor driver exist
+ drivers = \
+ {driver['id']: driver for driver in self.congress.list_drivers()['results']}
+ if self.doctor_driver not in drivers:
+ raise Exception('Do not support doctor driver in congress')
+
+ self.policy_rules = \
+ {rule['name']: rule for rule in
+ self.congress.list_policy_rules(self.policy)['results']}
+
+ def get_inspector_url(self):
+ ds = self.congress.list_datasources()['results']
+ doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+ None)
+ congress_endpoint = self.congress.httpclient.get_endpoint(auth=self.auth)
+ return ('%s/v1/data-sources/%s/tables/events/rows' %
+ (congress_endpoint, doctor_ds['id']))
+
+ def start(self):
+ self.log.info('congress inspector start......')
+
+ for rule_name, rule in self.rules.items():
+ self._add_rule(rule_name, rule)
+
+ def stop(self):
+ self.log.info('congress inspector stop......')
+
+ for rule_name in self.rules.keys():
+ self._del_rule(rule_name)
+
+ def _add_rule(self, rule_name, rule):
+ if rule_name not in self.policy_rules:
+ self.congress.create_policy_rule(self.policy,
+ body={'name': rule_name,
+ 'rule': rule})
+
+ def _del_rule(self, rule_name):
+ if rule_name in self.policy_rules:
+ rule_id = self.policy_rules[rule_name]['id']
+ self.congress.delete_policy_rule(self.policy, rule_id)
diff --git a/doctor_tests/inspector/sample.py b/doctor_tests/inspector/sample.py
new file mode 100644
index 00000000..114e4ebd
--- /dev/null
+++ b/doctor_tests/inspector/sample.py
@@ -0,0 +1,169 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import collections
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from doctor_tests.common import utils
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import nova_client
+from doctor_tests.os_clients import neutron_client
+from doctor_tests.inspector.base import BaseInspector
+
+
+class SampleInspector(BaseInspector):
+ event_type = 'compute.host.down'
+
+ def __init__(self, conf, log):
+ super(SampleInspector, self).__init__(conf, log)
+ self.inspector_url = self.get_inspector_url()
+ self.novaclients = list()
+ self._init_novaclients()
+ # Normally we use this client for non redundant API calls
+ self.nova = self.novaclients[0]
+
+ auth = get_identity_auth(project=self.conf.doctor_project)
+ session = get_session(auth=auth)
+ self.neutron = neutron_client(session)
+
+ self.servers = collections.defaultdict(list)
+ self.hostnames = list()
+ self.app = None
+
+ def _init_novaclients(self):
+ self.NUMBER_OF_CLIENTS = self.conf.instance_count
+ auth = get_identity_auth(project=self.conf.doctor_project)
+ session = get_session(auth=auth)
+ for i in range(self.NUMBER_OF_CLIENTS):
+ self.novaclients.append(
+ nova_client(self.conf.nova_version, session))
+
+ def _init_servers_list(self):
+ self.servers.clear()
+ opts = {'all_tenants': True}
+ servers = self.nova.servers.list(search_opts=opts)
+ for server in servers:
+ try:
+ host = server.__dict__.get('OS-EXT-SRV-ATTR:host')
+ self.servers[host].append(server)
+ self.log.debug('get hostname=%s from server=%s' % (host, server))
+ except Exception as e:
+ self.log.info('can not get hostname from server=%s' % server)
+
+ def get_inspector_url(self):
+ return 'http://%s:%s' % (self.conf.inspector.ip, self.conf.inspector.port)
+
+ def start(self):
+ self.log.info('sample inspector start......')
+ self._init_servers_list()
+ self.app = InspectorApp(self.conf.inspector.port, self, self.log)
+ self.app.start()
+
+ def stop(self):
+ self.log.info('sample inspector stop......')
+ if not self.app:
+ return
+ for hostname in self.hostnames:
+ self.nova.services.force_down(hostname, 'nova-compute', False)
+
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ }
+ url = '%s%s' % (self.inspector_url, 'shutdown') \
+ if self.inspector_url.endswith('/') else \
+ '%s%s' % (self.inspector_url, '/shutdown')
+ requests.post(url, data='', headers=headers)
+
+ def handle_events(self, events):
+ for event in events:
+ hostname = event['details']['hostname']
+ event_type = event['type']
+ if event_type == self.event_type:
+ self.hostnames.append(hostname)
+ thr1 = self._disable_compute_host(hostname)
+ thr2 = self._vms_reset_state('error', hostname)
+ thr3 = self._set_ports_data_plane_status('DOWN', hostname)
+ thr1.join()
+ thr2.join()
+ thr3.join()
+
+ @utils.run_async
+ def _disable_compute_host(self, hostname):
+ self.nova.services.force_down(hostname, 'nova-compute', True)
+ self.log.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
+
+ @utils.run_async
+ def _vms_reset_state(self, state, hostname):
+
+ @utils.run_async
+ def _vm_reset_state(nova, server, state):
+ nova.servers.reset_state(server, state)
+ self.log.info('doctor mark vm(%s) error at %s' % (server, time.time()))
+
+ thrs = []
+ for nova, server in zip(self.novaclients, self.servers[hostname]):
+ t = _vm_reset_state(nova, server, state)
+ thrs.append(t)
+ for t in thrs:
+ t.join()
+
+ @utils.run_async
+ def _set_ports_data_plane_status(self, status, hostname):
+ body = {'data_plane_status': status}
+
+ @utils.run_async
+ def _set_port_data_plane_status(port_id):
+ self.neutron.update_port(port_id, body)
+ self.log.info('doctor set data plane status %s on port %s' % (status, port_id))
+
+ thrs = []
+ params = {'binding:host_id': hostname}
+ for port_id in self.neutron.list_ports(**params):
+ t = _set_port_data_plane_status(port_id)
+ thrs.append(t)
+ for t in thrs:
+ t.join()
+
+
+class InspectorApp(Thread):
+
+ def __init__(self, port, inspector, log):
+ Thread.__init__(self)
+ self.port = port
+ self.inspector = inspector
+ self.log = log
+
+ def run(self):
+ app = Flask('inspector')
+
+ @app.route('/events', methods=['PUT'])
+ def event_posted():
+ self.log.info('event posted in sample inspector at %s' % time.time())
+ self.log.info('sample inspector = %s' % self.inspector)
+ self.log.info('sample inspector received data = %s' % request.data)
+ events = json.loads(request.data.decode('utf8'))
+ self.inspector.handle_events(events)
+ return "OK"
+
+ @app.route('/shutdown', methods=['POST'])
+ def shutdown():
+ self.log.info('shutdown inspector app server at %s' % time.time())
+ func = request.environ.get('werkzeug.server.shutdown')
+ if func is None:
+ raise RuntimeError('Not running with the Werkzeug Server')
+ func()
+ return 'inspector app shutting down...'
+
+ app.run(host="0.0.0.0", port=self.port)
diff --git a/doctor_tests/installer/__init__.py b/doctor_tests/installer/__init__.py
new file mode 100644
index 00000000..02735b11
--- /dev/null
+++ b/doctor_tests/installer/__init__.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+ cfg.StrOpt('type',
+ default=os.environ.get('INSTALLER_TYPE', 'local'),
+ choices=['local', 'apex'],
+ help='the type of installer',
+ required=True),
+ cfg.StrOpt('ip',
+ default=os.environ.get('INSTALLER_IP', '127.0.0.1'),
+ help='the ip of installer'),
+ cfg.StrOpt('username',
+ default='root',
+ help='the user name for login installer server',
+ required=True),
+]
+
+
+_installer_name_class_mapping = {
+ 'local': 'doctor_tests.installer.local.LocalInstaller',
+ 'apex': 'doctor_tests.installer.apex.ApexInstaller'
+}
+
+
+def get_installer(conf, log):
+ installer_class = _installer_name_class_mapping[conf.installer.type]
+ return importutils.import_object(installer_class, conf, log)
diff --git a/doctor_tests/installer/apex.py b/doctor_tests/installer/apex.py
new file mode 100644
index 00000000..2a1ce94b
--- /dev/null
+++ b/doctor_tests/installer/apex.py
@@ -0,0 +1,126 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import getpass
+import grp
+import os
+import pwd
+import stat
+import subprocess
+
+from doctor_tests.common.utils import SSHClient
+from doctor_tests.installer.base import BaseInstaller
+
+
+class ApexInstaller(BaseInstaller):
+ node_user_name = 'heat-admin'
+ cm_set_script = 'set_ceilometer.py'
+ cm_restore_script = 'restore_ceilometer.py'
+
+ def __init__(self, conf, log):
+ super(ApexInstaller, self).__init__(conf, log)
+ self.client = SSHClient(self.conf.installer.ip,
+ self.conf.installer.username,
+ look_for_keys=True)
+ self.key_file = None
+ self.controllers = list()
+ self.controller_clients = list()
+ self.servers = list()
+
+ def setup(self):
+ self.log.info('Setup Apex installer start......')
+
+ self.get_ssh_key_from_installer()
+ self.get_controller_ips()
+ self.set_apply_patches()
+ self.setup_stunnel()
+
+ def cleanup(self):
+ self.restore_apply_patches()
+ for server in self.servers:
+ server.terminate()
+
+ def get_ssh_key_from_installer(self):
+ self.log.info('Get SSH keys from Apex installer......')
+
+ if self.key_file is not None:
+ self.log.info('Already have SSH keys from Apex installer......')
+ return self.key_file
+
+ self.client.scp('/home/stack/.ssh/id_rsa', './instack_key', method='get')
+ user = getpass.getuser()
+ uid = pwd.getpwnam(user).pw_uid
+ gid = grp.getgrnam(user).gr_gid
+ os.chown('./instack_key', uid, gid)
+ os.chmod('./instack_key', stat.S_IREAD)
+ current_dir = os.curdir
+ self.key_file = '{0}/{1}'.format(current_dir, 'instack_key')
+ return self.key_file
+
+ def get_controller_ips(self):
+ self.log.info('Get controller ips from Apex installer......')
+
+ command = "source stackrc; " \
+ "nova list | grep ' overcloud-controller-[0-9] ' " \
+ "| sed -e 's/^.*ctlplane=//' |awk '{print $1}'"
+ ret, controllers = self.client.ssh(command)
+ if ret:
+ raise Exception('Exec command to get controller ips in Apex installer failed'
+ 'ret=%s, output=%s' % (ret, controllers))
+ self.log.info('Get controller_ips:%s from Apex installer' % controllers)
+ self.controllers = controllers
+
+ def get_host_ip_from_hostname(self, hostname):
+ self.log.info('Get host ip from host name in Apex installer......')
+
+ hostname_in_undercloud = hostname.split('.')[0]
+
+ command = "source stackrc; nova show %s | awk '/ ctlplane network /{print $5}'" % (hostname_in_undercloud)
+ ret, host_ip = self.client.ssh(command)
+ if ret:
+ raise Exception('Exec command to get host ip from hostname(%s) in Apex installer failed'
+ 'ret=%s, output=%s' % (hostname, ret, host_ip))
+ self.log.info('Get host_ip:%s from host_name:%s in Apex installer' % (host_ip, hostname))
+ return host_ip[0]
+
+ def setup_stunnel(self):
+ self.log.info('Setup ssh stunnel in controller nodes in Apex installer......')
+ for node_ip in self.controllers:
+ cmd = "sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s -R %s:localhost:%s sleep 600 > ssh_tunnel.%s.log 2>&1 < /dev/null &" \
+ % (self.key_file, self.node_user_name, node_ip,
+ self.conf.consumer.port, self.conf.consumer.port, node_ip)
+ server = subprocess.Popen(cmd, shell=True)
+ self.servers.append(server)
+ server.communicate()
+
+ def set_apply_patches(self):
+ self.log.info('Set apply patches start......')
+
+ for node_ip in self.controllers:
+ client = SSHClient(node_ip, self.node_user_name, key_filename=self.key_file)
+ self.controller_clients.append(client)
+ self._ceilometer_apply_patches(client, self.cm_set_script)
+
+ def restore_apply_patches(self):
+ self.log.info('restore apply patches start......')
+
+ for client in self.controller_clients:
+ self._ceilometer_apply_patches(client, self.cm_restore_script)
+
+ def _ceilometer_apply_patches(self, ssh_client, script_name):
+ installer_dir = os.path.dirname(os.path.realpath(__file__))
+ script_abs_path = '{0}/{1}/{2}'.format(installer_dir, 'common', script_name)
+
+ ssh_client.scp(script_abs_path, script_name)
+ cmd = 'sudo python %s' % script_name
+ ret, output = ssh_client.ssh(cmd)
+ if ret:
+ raise Exception('Do the ceilometer command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cmd, output))
+ ssh_client.ssh('sudo systemctl restart openstack-ceilometer-notification.service')
+
diff --git a/doctor_tests/installer/base.py b/doctor_tests/installer/base.py
new file mode 100644
index 00000000..fa39816a
--- /dev/null
+++ b/doctor_tests/installer/base.py
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInstaller(object):
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractproperty
+ def node_user_name(self):
+ """user name for login to cloud node"""
+
+ @abc.abstractmethod
+ def get_ssh_key_from_installer(self):
+ pass
+
+ @abc.abstractmethod
+ def get_host_ip_from_hostname(self, hostname):
+ pass
+
+ @abc.abstractmethod
+ def setup(self):
+ pass
+
+ @abc.abstractmethod
+ def cleanup(self):
+ pass
diff --git a/doctor_tests/installer/common/congress.py b/doctor_tests/installer/common/congress.py
new file mode 100644
index 00000000..db882de2
--- /dev/null
+++ b/doctor_tests/installer/common/congress.py
@@ -0,0 +1,47 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+def set_doctor_driver_conf(ssh_client, restart_cmd):
+ cg_set_cmd = '''#!/bin/bash
+co_conf=/etc/congress/congress.conf
+co_conf_bak=/etc/congress/congress.conf.bak
+co_entry="congress.datasources.doctor_driver.DoctorDriver"
+if sudo grep -e "^drivers.*$co_entry" $co_conf; then
+ echo "NOTE: congress is configured as we needed"
+else
+ echo "modify the congress config"
+ sudo cp $co_conf $co_conf_bak
+ sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf
+ %s
+fi
+ ''' % (restart_cmd)
+
+ ret, output = ssh_client.ssh(cg_set_cmd)
+ if ret:
+ raise Exception('Do the congress command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cg_set_cmd, output))
+
+
+def restore_doctor_driver_conf(ssh_client, restart_cmd):
+ cg_restore_cmd = '''#!/bin/bash
+co_conf=/etc/congress/congress.conf
+co_conf_bak=/etc/congress/congress.conf.bak
+if [ -e $co_conf_bak ]; then
+ echo "restore the congress config"
+ sudo cp $co_conf_bak $co_conf
+ sudo rm $co_conf_bak
+ %s
+else
+ echo "Do not need to restore the congress config"
+fi
+ ''' % (restart_cmd)
+
+ ret, output = ssh_client.ssh(cg_restore_cmd)
+ if ret:
+ raise Exception('Do the congress command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cg_restore_cmd, output))
diff --git a/doctor_tests/installer/common/restore_ceilometer.py b/doctor_tests/installer/common/restore_ceilometer.py
new file mode 100644
index 00000000..d25b9ede
--- /dev/null
+++ b/doctor_tests/installer/common/restore_ceilometer.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+
+
+def restore_ep_config():
+
+ if not os.path.isfile(ep_file_bak):
+ print('Bak_file:%s does not exist.' % ep_file_bak)
+ else:
+ print('restore')
+ shutil.copyfile(ep_file_bak, ep_file)
+ os.remove(ep_file_bak)
+ return
+
+
+restore_ep_config()
diff --git a/doctor_tests/installer/common/set_ceilometer.py b/doctor_tests/installer/common/set_ceilometer.py
new file mode 100644
index 00000000..f5946cb2
--- /dev/null
+++ b/doctor_tests/installer/common/set_ceilometer.py
@@ -0,0 +1,44 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+import yaml
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+event_notifier_topic = 'notifier://?topic=alarm.all'
+
+
+def set_notifier_topic():
+ config_modified = False
+
+ if not os.path.isfile(ep_file):
+ raise Exception("File doesn't exist: %s." % ep_file)
+
+ with open(ep_file, 'r') as file:
+ config = yaml.safe_load(file)
+
+ sinks = config['sinks']
+ for sink in sinks:
+ if sink['name'] == 'event_sink':
+ publishers = sink['publishers']
+ if event_notifier_topic not in publishers:
+ print('Add event notifier in ceilometer')
+ publishers.append(event_notifier_topic)
+ config_modified = True
+ else:
+ print('NOTE: event notifier is configured in ceilometer as we needed')
+
+ if config_modified:
+ shutil.copyfile(ep_file, ep_file_bak)
+ with open(ep_file, 'w+') as file:
+ file.write(yaml.safe_dump(config))
+
+
+set_notifier_topic()
diff --git a/doctor_tests/installer/local.py b/doctor_tests/installer/local.py
new file mode 100644
index 00000000..7d0ae542
--- /dev/null
+++ b/doctor_tests/installer/local.py
@@ -0,0 +1,109 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+import subprocess
+
+from doctor_tests.installer.base import BaseInstaller
+from doctor_tests.common.utils import load_json_file
+from doctor_tests.common.utils import write_json_file
+
+
+class LocalInstaller(BaseInstaller):
+ node_user_name = 'root'
+
+ nova_policy_file = '/etc/nova/policy.json'
+ nova_policy_file_backup = '%s%s' % (nova_policy_file, '.bak')
+
+ def __init__(self, conf, log):
+ super(LocalInstaller, self).__init__(conf, log)
+ self.policy_modified = False
+ self.add_policy_file = False
+
+ def setup(self):
+ self.get_ssh_key_from_installer()
+ self.set_apply_patches()
+
+ def cleanup(self):
+ self.restore_apply_patches()
+
+ def get_ssh_key_from_installer(self):
+ self.log.info('Assuming SSH keys already exchanged with computer for local installer type')
+ return None
+
+ def get_host_ip_from_hostname(self, hostname):
+ self.log.info('Get host ip from host name in local installer......')
+
+ cmd = "getent hosts %s | awk '{ print $1 }'" % (hostname)
+ server = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+ stdout, stderr = server.communicate()
+ host_ip = stdout.strip()
+
+ self.log.info('Get host_ip:%s from host_name:%s in local installer' % (host_ip, hostname))
+ return host_ip
+
+ def set_apply_patches(self):
+ self._set_nova_policy()
+
+ def restore_apply_patches(self):
+ self._restore_nova_policy()
+
+ def _set_nova_policy(self):
+ host_status_policy = 'os_compute_api:servers:show:host_status'
+ host_status_rule = 'rule:admin_or_owner'
+ policy_data = {
+ 'context_is_admin': 'role:admin',
+ 'owner': 'user_id:%(user_id)s',
+ 'admin_or_owner': 'rule:context_is_admin or rule:owner',
+ host_status_policy: host_status_rule
+ }
+
+ if os.path.isfile(self.nova_policy_file):
+ data = load_json_file(self.nova_policy_file)
+ if host_status_policy in data:
+ rule_origion = data[host_status_policy]
+ if host_status_rule == rule_origion:
+ self.log.info('Do not need to modify nova policy.')
+ self.policy_modified = False
+ else:
+ # update the host_status_policy
+ data[host_status_policy] = host_status_rule
+ self.policy_modified = True
+ else:
+ # add the host_status_policy, if the admin_or_owner is not
+ # defined, add it also
+ for policy, rule in policy_data.items():
+ if policy not in data:
+ data[policy] = rule
+ self.policy_modified = True
+ if self.policy_modified:
+ self.log.info('Nova policy is Modified.')
+ shutil.copyfile(self.nova_policy_file,
+ self.nova_policy_file_backup)
+ else:
+ # file does not exit, create a new one and add the policy
+ self.log.info('Nova policy file not exist. Creating a new one')
+ data = policy_data
+ self.add_policy_file = True
+
+ if self.policy_modified or self.add_policy_file:
+ write_json_file(self.nova_policy_file, data)
+ os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"')
+
+ def _restore_nova_policy(self):
+ if self.policy_modified:
+ shutil.copyfile(self.nova_policy_file_backup, self.nova_policy_file)
+ os.remove(self.nova_policy_file_backup)
+ elif self.add_policy_file:
+ os.remove(self.nova_policy_file)
+
+ if self.add_policy_file or self.policy_modified:
+ os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"')
+ self.add_policy_file = False
+ self.policy_modified = False
diff --git a/doctor_tests/instance.py b/doctor_tests/instance.py
new file mode 100644
index 00000000..27f412e2
--- /dev/null
+++ b/doctor_tests/instance.py
@@ -0,0 +1,114 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import time
+
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import neutron_client
+from doctor_tests.os_clients import nova_client
+
+OPTS = [
+ cfg.StrOpt('flavor',
+ default='m1.tiny',
+ help='the name of flavor',
+ required=True),
+ cfg.IntOpt('instance_count',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the count of instance',
+ required=True),
+ cfg.StrOpt('instance_basename',
+ default='doctor_vm',
+ help='the base name of instance',
+ required=True),
+]
+
+
+class Instance(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(username=self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ project=self.conf.doctor_project)
+ self.nova = \
+ nova_client(conf.nova_version,
+ get_session(auth=self.auth))
+ self.neutron = neutron_client(get_session(auth=self.auth))
+ self.servers = {}
+ self.vm_names = []
+
+ def create(self):
+ self.log.info('instance create start......')
+
+ # get flavor, image and network for vm boot
+ flavors = {flavor.name: flavor for flavor in self.nova.flavors.list()}
+ flavor = flavors.get(self.conf.flavor)
+ image = self.nova.glance.find_image(self.conf.image_name)
+ network = self.neutron.list_networks(name=self.conf.net_name)['networks'][0]
+ nics = {'net-id': network['id']}
+
+ self.servers = \
+ {getattr(server, 'name'): server
+ for server in self.nova.servers.list()}
+ for i in range(0, self.conf.instance_count):
+ vm_name = "%s%d"%(self.conf.instance_basename, i)
+ self.vm_names.append(vm_name)
+ if vm_name not in self.servers:
+ server = self.nova.servers.create(vm_name, image,
+ flavor, nics=[nics])
+ self.servers[vm_name] = server
+ time.sleep(0.1)
+
+ self.log.info('instance create end......')
+
+ def delete(self):
+ self.log.info('instance delete start.......')
+
+ for vm_name in self.vm_names:
+ if vm_name in self.servers:
+ self.nova.servers.delete(self.servers[vm_name])
+ time.sleep(0.1)
+
+ # check that all vms are deleted
+ while self.nova.servers.list():
+ time.sleep(0.1)
+ self.servers.clear()
+ del self.vm_names[:]
+
+ self.log.info('instance delete end.......')
+
+ def wait_for_vm_launch(self):
+ self.log.info('wait for vm launch start......')
+
+ wait_time = 60
+ count = 0
+ while count < wait_time:
+ active_count = 0
+ for vm_name in self.vm_names:
+ server = self.nova.servers.get(self.servers[vm_name])
+ server_status = getattr(server, 'status').lower()
+ if 'active' == server_status:
+ active_count += 1
+ elif 'error' == server_status:
+ raise Exception('vm launched with error state')
+ else:
+ time.sleep(2)
+ count += 1
+ continue
+ if active_count == self.conf.instance_count:
+ self.log.info('wait for vm launch end......')
+ return
+ count += 1
+ time.sleep(2)
+ raise Exception('time out for vm launch')
+
diff --git a/doctor_tests/logger.py b/doctor_tests/logger.py
new file mode 100644
index 00000000..b7a49fdb
--- /dev/null
+++ b/doctor_tests/logger.py
@@ -0,0 +1,46 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+# Usage:
+# import doctor_logger
+# logger = doctor_logger.Logger("script_name").getLogger()
+# logger.info("message to be shown with - INFO - ")
+# logger.debug("message to be shown with - DEBUG -")
+
+import logging
+import os
+
+
+class Logger(object):
+ def __init__(self, logger_name):
+
+ CI_DEBUG = os.getenv('CI_DEBUG')
+
+ self.logger = logging.getLogger(logger_name)
+ self.logger.propagate = 0
+ self.logger.setLevel(logging.DEBUG)
+
+ formatter = logging.Formatter('%(asctime)s %(filename)s %(lineno)d '
+ '%(levelname)-6s %(message)s')
+
+ ch = logging.StreamHandler()
+ ch.setFormatter(formatter)
+ if CI_DEBUG is not None and CI_DEBUG.lower() == "true":
+ ch.setLevel(logging.DEBUG)
+ else:
+ ch.setLevel(logging.INFO)
+ self.logger.addHandler(ch)
+
+ filename = '%s.log' % logger_name
+ file_handler = logging.FileHandler(filename, mode='w')
+ file_handler.setFormatter(formatter)
+ file_handler.setLevel(logging.DEBUG)
+ self.logger.addHandler(file_handler)
+
+ def getLogger(self):
+ return self.logger
diff --git a/doctor_tests/main.py b/doctor_tests/main.py
new file mode 100644
index 00000000..006aac9f
--- /dev/null
+++ b/doctor_tests/main.py
@@ -0,0 +1,215 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+from os.path import isfile, join
+import random
+import sys
+import time
+
+from doctor_tests.alarm import Alarm
+from doctor_tests.common.constants import Host
+from doctor_tests.common.utils import match_rep_in_file
+from doctor_tests import config
+from doctor_tests.consumer import get_consumer
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.image import Image
+from doctor_tests.instance import Instance
+from doctor_tests.inspector import get_inspector
+from doctor_tests.installer import get_installer
+import doctor_tests.logger as doctor_log
+from doctor_tests.network import Network
+from doctor_tests.monitor import get_monitor
+from doctor_tests.os_clients import nova_client
+from doctor_tests.profiler_poc import main as profiler_main
+from doctor_tests.scenario.common import calculate_notification_time
+from doctor_tests.scenario.network_failure import NetworkFault
+from doctor_tests.user import User
+
+
+LOG = doctor_log.Logger('doctor').getLogger()
+
+
+class DoctorTest(object):
+
+ def __init__(self, conf):
+ self.conf = conf
+ self.image = Image(self.conf, LOG)
+ self.user = User(self.conf, LOG)
+ self.network = Network(self.conf, LOG)
+ self.instance = Instance(self.conf, LOG)
+ self.alarm = Alarm(self.conf, LOG)
+ self.installer = get_installer(self.conf, LOG)
+ self.inspector = get_inspector(self.conf, LOG)
+ self.monitor = get_monitor(self.conf,
+ self.inspector.get_inspector_url(),
+ LOG)
+ self.consumer = get_consumer(self.conf, LOG)
+ self.fault = NetworkFault(self.conf, self.installer, LOG)
+ auth = get_identity_auth(project=self.conf.doctor_project)
+ self.nova = nova_client(self.conf.nova_version,
+ get_session(auth=auth))
+ self.down_host = None
+
+ def setup(self):
+ # prepare the cloud env
+ self.installer.setup()
+
+ # preparing VM image...
+ self.image.create()
+
+ # creating test user...
+ self.user.create()
+ self.user.update_quota()
+
+ # creating VM...
+ self.network.create()
+ self.instance.create()
+ self.instance.wait_for_vm_launch()
+
+ # creating alarm...
+ self.alarm.create()
+
+ # starting doctor sample components...
+ self.inspector.start()
+
+ self.down_host = self.get_host_info_for_random_vm()
+ self.monitor.start(self.down_host)
+
+ self.consumer.start()
+
+ def run(self):
+ """run doctor test"""
+ try:
+ LOG.info('doctor test starting.......')
+
+ # prepare test env
+ self.setup()
+
+ # wait for aodh alarms are updated in caches for event evaluator,
+ # sleep time should be larger than event_alarm_cache_ttl(default 60)
+ time.sleep(60)
+
+ # injecting host failure...
+ # NOTE (umar) add INTERFACE_NAME logic to host injection
+
+ self.fault.start(self.down_host)
+ time.sleep(10)
+
+ # verify the test results
+ # NOTE (umar) copy remote monitor.log file when monitor=collectd
+ self.check_host_status(self.down_host.name, 'down')
+
+ notification_time = calculate_notification_time()
+ if notification_time < 1 and notification_time > 0:
+ LOG.info('doctor test successfully, notification_time=%s' % notification_time)
+ else:
+ LOG.error('doctor test failed, notification_time=%s' % notification_time)
+ sys.exit(1)
+
+ if self.conf.profiler_type:
+ LOG.info('doctor test begin to run profile.......')
+ self.collect_logs()
+ self.run_profiler()
+ except Exception as e:
+ LOG.error('doctor test failed, Exception=%s' % e)
+ sys.exit(1)
+ finally:
+ self.cleanup()
+
+ def get_host_info_for_random_vm(self):
+ num = random.randint(0, self.conf.instance_count - 1)
+ vm_name = "%s%d" % (self.conf.instance_basename, num)
+
+ servers = \
+ {getattr(server, 'name'): server
+ for server in self.nova.servers.list()}
+ server = servers.get(vm_name)
+ if not server:
+ raise \
+ Exception('Can not find instance: vm_name(%s)' % vm_name)
+ host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname')
+ host_ip = self.installer.get_host_ip_from_hostname(host_name)
+
+ LOG.info('Get host info(name:%s, ip:%s) which vm(%s) launched at'
+ % (host_name, host_ip, vm_name))
+ return Host(host_name, host_ip)
+
+ def check_host_status(self, hostname, state):
+ service = self.nova.services.list(host=hostname, binary='nova-compute')
+ host_state = service[0].__dict__.get('state')
+ assert host_state == state
+
+ def unset_forced_down_hosts(self):
+ if self.down_host:
+ self.nova.services.force_down(self.down_host.name, 'nova-compute', False)
+ time.sleep(2)
+ self.check_host_status(self.down_host.name, 'up')
+
+ def collect_logs(self):
+ self.fault.get_disable_network_log()
+
+ def run_profiler(self):
+
+ log_file = '{0}/{1}'.format(sys.path[0], 'disable_network.log')
+ reg = '(?<=doctor set link down at )\d+.\d+'
+ linkdown = float(match_rep_in_file(reg, log_file).group(0))
+
+ log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log')
+ reg = '(.* doctor mark vm.* error at )(\d+.\d+)'
+ vmdown = float(match_rep_in_file(reg, log_file).group(2))
+
+ reg = '(?<=doctor mark host.* down at )\d+.\d+'
+ hostdown = float(match_rep_in_file(reg, log_file).group(2))
+
+ reg = '(?<=doctor monitor detected at )\d+.\d+'
+ detected = float(match_rep_in_file(reg, log_file).group(0))
+
+ reg = '(?<=doctor consumer notified at )\d+.\d+'
+ notified = float(match_rep_in_file(reg, log_file).group(0))
+
+ # TODO(yujunz) check the actual delay to verify time sync status
+ # expected ~1s delay from $trigger to $linkdown
+ relative_start = linkdown
+ os.environ['DOCTOR_PROFILER_T00'] = str(int((linkdown - relative_start)*1000))
+ os.environ['DOCTOR_PROFILER_T01'] = str(int((detected - relative_start) * 1000))
+ os.environ['DOCTOR_PROFILER_T03'] = str(int((vmdown - relative_start) * 1000))
+ os.environ['DOCTOR_PROFILER_T04'] = str(int((hostdown - relative_start) * 1000))
+ os.environ['DOCTOR_PROFILER_T09'] = str(int((notified - relative_start) * 1000))
+
+ profiler_main(log=LOG)
+
+ def cleanup(self):
+ self.unset_forced_down_hosts()
+ self.inspector.stop()
+ self.monitor.stop()
+ self.consumer.stop()
+ self.installer.cleanup()
+ self.alarm.delete()
+ self.instance.delete()
+ self.network.delete()
+ self.image.delete()
+ self.fault.cleanup()
+ self.user.delete()
+
+
+def main():
+ """doctor main"""
+ test_dir = os.path.split(os.path.realpath(__file__))[0]
+ doctor_root_dir = os.path.dirname(test_dir)
+
+ config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/')
+ config_files = [join(config_file_dir, f) for f in os.listdir(config_file_dir)
+ if isfile(join(config_file_dir, f))]
+
+ conf = config.prepare_conf(args=sys.argv[1:],
+ config_files=config_files)
+
+ doctor = DoctorTest(conf)
+ doctor.run()
diff --git a/doctor_tests/monitor/__init__.py b/doctor_tests/monitor/__init__.py
new file mode 100644
index 00000000..7e30c9f8
--- /dev/null
+++ b/doctor_tests/monitor/__init__.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+ cfg.StrOpt('type',
+ default='sample',
+ choices=['sample', 'collectd'],
+ help='the type of doctor monitor component',
+ required=True),
+]
+
+
+_monitor_name_class_mapping = {
+ 'sample': 'doctor_tests.monitor.sample.SampleMonitor',
+ 'collectd': 'doctor_tests.monitor.collectd.CollectdMonitor'
+}
+
+def get_monitor(conf, inspector_url, log):
+ monitor_class = _monitor_name_class_mapping.get(conf.monitor.type)
+ return importutils.import_object(monitor_class, conf,
+ inspector_url, log)
diff --git a/doctor_tests/monitor/base.py b/doctor_tests/monitor/base.py
new file mode 100644
index 00000000..119c8a1c
--- /dev/null
+++ b/doctor_tests/monitor/base.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseMonitor(object):
+ """Monitor computer fault and report error to the inspector"""
+ def __init__(self, conf, inspector_url, log):
+ self.conf = conf
+ self.log = log
+ self.inspector_url = inspector_url
+
+ @abc.abstractmethod
+ def start(self, host):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass
diff --git a/doctor_tests/monitor/collectd.py b/doctor_tests/monitor/collectd.py
new file mode 100644
index 00000000..a22d7edc
--- /dev/null
+++ b/doctor_tests/monitor/collectd.py
@@ -0,0 +1,137 @@
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import socket
+import getpass
+import sys
+
+from doctor_tests.monitor.base import BaseMonitor
+
+
+class CollectdMonitor(BaseMonitor):
+ def __init__(self, conf, inspector_url, log):
+ super(CollectdMonitor, self).__init__(conf, inspector_url, log)
+ self.top_dir = os.path.dirname(sys.path[0])
+ tmp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ tmp_sock.connect(("8.8.8.8", 80))
+
+ ## control_ip is the IP of primary interface of control node i.e.
+ ## eth0, eno1. It is used by collectd monitor to communicate with
+ ## sample inspector.
+ ## TODO (umar) see if mgmt IP of control is a better option. Also
+ ## primary interface may not be the right option
+ self.control_ip = tmp_sock.getsockname()[0]
+ self.compute_user = getpass.getuser()
+ self.interface_name = os.environ.get('INTERFACE_NAME') or ''
+ self.inspector_type = os.environ.get('INSPECTOR_TYPE', 'sample')
+ self.auth_url = os.environ.get('OS_AUTH_URL')
+ self.username = os.environ.get('OS_USERNAME')
+ self.password = os.environ.get('OS_PASSWORD')
+ self.project_name = os.environ.get('OS_PROJECT_NAME')
+ self.user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default'
+ self.user_domain_id = os.environ.get('OS_USER_DOMAIN_ID')
+ self.project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default'
+ self.project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID')
+ self.ssh_opts_cpu = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+
+ def start(self, host):
+ self.log.info("Collectd monitor start.........")
+ self.compute_host = host.name
+ self.compute_ip = host.ip
+ f = open("%s/tests/collectd.conf" % self.top_dir, 'w')
+ collectd_conf_file = """
+Hostname %s
+FQDNLookup false
+Interval 1
+MaxReadInterval 2
+
+<LoadPlugin python>
+Globals true
+</LoadPlugin>
+LoadPlugin ovs_events
+LoadPlugin logfile
+
+<Plugin logfile>
+ File \"/var/log/collectd.log\"
+ Timestamp true
+ LogLevel \"info\"
+</Plugin>
+
+<Plugin python>
+ ModulePath \"/home/%s\"
+ LogTraces true
+ Interactive false
+ Import \"collectd_plugin\"
+ <Module \"collectd_plugin\">
+ control_ip \"%s\"
+ compute_ip \"%s\"
+ compute_host \"%s\"
+ compute_user \"%s\"
+ inspector_type \"%s\"
+ os_auth_url \"%s\"
+ os_username \"%s\"
+ os_password \"%s\"
+ os_project_name \"%s\"
+ os_user_domain_name \"%s\"
+ os_user_domain_id \"%s\"
+ os_project_domain_name \"%s\"
+ os_project_domain_id \"%s\"
+ </Module>
+</Plugin>
+
+<Plugin ovs_events>
+ Port 6640
+ Socket \"/var/run/openvswitch/db.sock\"
+ Interfaces \"@INTERFACE_NAME@\"
+ SendNotification true
+ DispatchValues false
+</Plugin>
+ """ % (self.compute_host, self.compute_user, self.control_ip, self.compute_ip, self.compute_host, self.compute_user,
+ self.inspector_type, self.auth_url, self.username, self.password, self.project_name, self.user_domain_name,
+ self.user_domain_id, self.project_domain_name, self.project_domain_id)
+ f.write(collectd_conf_file)
+ f.close()
+
+ os.system(" scp %s %s/tests/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+ self.log.info("after first scp")
+ ## @TODO (umar) Always assuming that the interface is assigned an IP if
+ ## interface name is not provided. See if there is a better approach
+ os.system(""" ssh %s %s@%s \"if [ -n \"%s\" ]; then
+ dev=%s
+ else
+ dev=\$(sudo ip a | awk '/ %s\//{print \$NF}')
+ fi
+ sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \$collectd_conf ]; then
+ sudo cp \$collectd_conf \${collectd_conf}-doctor-saved
+ else
+ sudo touch \${collectd_conf}-doctor-created
+ fi
+ sudo mv collectd.conf /opt/collectd/etc/collectd.conf\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip, self.interface_name, self.interface_name, self.compute_ip))
+ self.log.info("after first ssh")
+ os.system(" scp %s %s/tests/lib/monitors/collectd/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+ self.log.info("after sec scp")
+ os.system(" ssh %s %s@%s \"sudo pkill collectd; sudo /opt/collectd/sbin/collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+ self.log.info("after sec ssh")
+
+ def stop(self):
+ os.system(" ssh %s %s@%s \"sudo pkill collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+
+ def cleanup(self):
+ os.system(""" ssh %s %s@%s \"
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \"\${collectd_conf}-doctor-created\" ]; then
+ sudo rm \"\${collectd_conf}-doctor-created\"
+ sudo rm \$collectd_conf
+ elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then
+ sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf
+ sudo rm \"\${collectd_conf}-doctor-saved\"
+ fi\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+ os.remove("%s/tests/collectd.conf" % self.top_dir)
diff --git a/doctor_tests/monitor/sample.py b/doctor_tests/monitor/sample.py
new file mode 100644
index 00000000..7a463048
--- /dev/null
+++ b/doctor_tests/monitor/sample.py
@@ -0,0 +1,106 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from datetime import datetime
+import json
+import requests
+import socket
+from threading import Thread
+import time
+
+from doctor_tests.identity_auth import get_session
+from doctor_tests.monitor.base import BaseMonitor
+
+
+class SampleMonitor(BaseMonitor):
+ event_type = "compute.host.down"
+
+ def __init__(self, conf, inspector_url, log):
+ super(SampleMonitor, self).__init__(conf, inspector_url, log)
+ self.session = get_session()
+ self.pinger = None
+
+ def start(self, host):
+ self.log.info('sample monitor start......')
+ self.pinger = Pinger(host.name, host.ip, self, self.log)
+ self.pinger.start()
+
+ def stop(self):
+ self.log.info('sample monitor stop......')
+ if self.pinger is not None:
+ self.pinger.stop()
+ self.pinger.join()
+
+ def report_error(self, hostname):
+ self.log.info('sample monitor report error......')
+ data = [
+ {
+ 'id': 'monitor_sample_id1',
+ 'time': datetime.now().isoformat(),
+ 'type': self.event_type,
+ 'details': {
+ 'hostname': hostname,
+ 'status': 'down',
+ 'monitor': 'monitor_sample',
+ 'monitor_event_id': 'monitor_sample_event1'
+ },
+ },
+ ]
+
+ auth_token = self.session.get_token() if \
+ self.conf.inspector.type != 'sample' else None
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ 'X-Auth-Token': auth_token,
+ }
+
+ url = '%s%s' % (self.inspector_url, 'events') \
+ if self.inspector_url.endswith('/') else \
+ '%s%s' % (self.inspector_url, '/events')
+ requests.put(url, data=json.dumps(data), headers=headers)
+
+
+class Pinger(Thread):
+ interval = 0.1 # second
+ timeout = 0.1 # second
+ ICMP_ECHO_MESSAGE = bytes([0x08, 0x00, 0xf7, 0xff, 0x00, 0x00, 0x00, 0x00])
+
+ def __init__(self, host_name, host_ip, monitor, log):
+ Thread.__init__(self)
+ self.monitor = monitor
+ self.hostname = host_name
+ self.ip_addr = host_ip or socket.gethostbyname(self.hostname)
+ self.log = log
+ self._stopped = False
+
+ def run(self):
+ self.log.info("Starting Pinger host_name(%s), host_ip(%s)"
+ % (self.hostname, self.ip_addr))
+
+ sock = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_ICMP)
+ sock.settimeout(self.timeout)
+ while True:
+ if self._stopped:
+ return
+ try:
+ sock.sendto(self.ICMP_ECHO_MESSAGE, (self.ip_addr, 0))
+ sock.recv(4096)
+ except socket.timeout:
+ self.log.info("doctor monitor detected at %s" % time.time())
+ self.monitor.report_error(self.hostname)
+ self.log.info("ping timeout, quit monitoring...")
+ self._stopped = True
+ return
+ time.sleep(self.interval)
+
+ def stop(self):
+ self.log.info("Stopping Pinger host_name(%s), host_ip(%s)"
+ % (self.hostname, self.ip_addr))
+ self._stopped = True
diff --git a/doctor_tests/network.py b/doctor_tests/network.py
new file mode 100644
index 00000000..ee153e66
--- /dev/null
+++ b/doctor_tests/network.py
@@ -0,0 +1,68 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import neutron_client
+
+
+OPTS = [
+ cfg.StrOpt('net_name',
+ default='doctor_net',
+ help='the name of test net',
+ required=True),
+ cfg.StrOpt('net_cidr',
+ default='192.168.168.0/24',
+ help='the cidr of test subnet',
+ required=True),
+]
+
+
+class Network(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(username=self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ project=self.conf.doctor_project)
+ self.neutron = neutron_client(get_session(auth=self.auth))
+ self.net = None
+ self.subnet = None
+
+ def create(self):
+ self.log.info('network create start.......')
+ net_name = self.conf.net_name
+ networks = self.neutron.list_networks(name=net_name)['networks']
+ self.net = networks[0] if networks \
+ else self.neutron.create_network(
+ {'network': {'name': net_name}})['network']
+ self.log.info('network create end.......')
+
+ self.log.info('subnet create start.......')
+ subnets = self.neutron.list_subnets(network_id=self.net['id'])['subnets']
+ subnet_param = {'name': net_name, 'network_id': self.net['id'],
+ 'cidr': self.conf.net_cidr, 'ip_version': 4,
+ 'enable_dhcp': False}
+ self.subnet = subnets[0] if subnets \
+ else self.neutron.create_subnet(
+ {'subnet': subnet_param})['subnet']
+ self.log.info('subnet create end.......')
+
+ def delete(self):
+ self.log.info('subnet delete start.......')
+ if self.subnet:
+ self.neutron.delete_subnet(self.subnet['id'])
+ self.log.info('subnet delete end.......')
+
+ self.log.info('network delete start.......')
+ if self.net:
+ self.neutron.delete_network(self.net['id'])
+ self.log.info('network delete end.......')
diff --git a/doctor_tests/os_clients.py b/doctor_tests/os_clients.py
new file mode 100644
index 00000000..44fa3aad
--- /dev/null
+++ b/doctor_tests/os_clients.py
@@ -0,0 +1,50 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+import aodhclient.client as aodhclient
+from congressclient.v1 import client as congressclient
+import glanceclient.client as glanceclient
+from keystoneclient.v2_0 import client as ks_client
+from neutronclient.v2_0 import client as neutronclient
+import novaclient.client as novaclient
+
+
+OPTS = [
+ cfg.StrOpt('glance_version', default='2', help='glance version'),
+ cfg.StrOpt('nova_version', default='2.34', help='Nova version'),
+ cfg.StrOpt('aodh_version', default='2', help='aodh version'),
+]
+
+
+def glance_client(version, session):
+ return glanceclient.Client(version=version,
+ session=session)
+
+
+def keystone_client(session):
+ return ks_client.Client(session=session)
+
+
+def nova_client(version, session):
+ return novaclient.Client(version=version,
+ session=session)
+
+
+def neutron_client(session):
+ return neutronclient.Client(session=session)
+
+
+def aodh_client(version, session):
+ return aodhclient.Client(version, session=session)
+
+
+def congress_client(session):
+ return congressclient.Client(session=session,
+ service_type='policy')
diff --git a/doctor_tests/profiler_poc.py b/doctor_tests/profiler_poc.py
new file mode 100644
index 00000000..ea36eaeb
--- /dev/null
+++ b/doctor_tests/profiler_poc.py
@@ -0,0 +1,100 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+"""
+PoC of performance profiler for OPNFV doctor project
+
+Usage:
+
+Export environment variables to set timestamp at each checkpoint in millisecond.
+Valid check points are: DOCTOR_PROFILER_T{00-09}
+
+See also: https://goo.gl/98Osig
+"""
+
+import json
+import os
+
+from oslo_config import cfg
+
+
+OPTS = [
+ cfg.StrOpt('profiler_type',
+ default=os.environ.get('PROFILER_TYPE', 'poc'),
+ help='the type of installer'),
+]
+
+
+OUTPUT = 'doctor_profiling_output'
+PREFIX = 'DOCTOR_PROFILER'
+TOTAL_CHECK_POINTS = 10
+MODULE_CHECK_POINTS = ['T00', 'T01', 'T04', 'T05', 'T06', 'T09']
+TAG_FORMAT = "{:<5}"
+# Inspired by https://github.com/reorx/httpstat
+TEMPLATE = """
+Total time cost: {total}(ms)
+==============================================================================>
+ |Monitor|Inspector |Controller|Notifier|Evaluator |
+ |{M00} |{M01} |{M02} |{M03} |{M04} |
+ | | | | | | | | | |
+link down:{T00}| | | | | | | | |
+ raw failure:{T01}| | | | | | | |
+ found affected:{T02}| | | | | | |
+ set VM error:{T03}| | | | | |
+ marked host down:{T04}| | | | |
+ notified VM error:{T05} | | | |
+ transformed event:{T06}| | |
+ evaluated event:{T07}| |
+ fired alarm:{T08}|
+ received alarm:{T09}
+"""
+
+
+def main(log=None):
+ check_points = ["T{:02d}".format(i) for i in range(TOTAL_CHECK_POINTS)]
+ module_map = {"M{:02d}".format(i):
+ (MODULE_CHECK_POINTS[i], MODULE_CHECK_POINTS[i + 1])
+ for i in range(len(MODULE_CHECK_POINTS) - 1)}
+
+ # check point tags
+ elapsed_ms = {cp: os.getenv("{}_{}".format(PREFIX, cp))
+ for cp in check_points}
+
+ def format_tag(tag):
+ return TAG_FORMAT.format(tag or '?')
+
+ tags = {cp: format_tag(ms) for cp, ms in elapsed_ms.items()}
+
+ def time_cost(cp):
+ if elapsed_ms[cp[0]] and elapsed_ms[cp[1]]:
+ return int(elapsed_ms[cp[1]]) - int(elapsed_ms[cp[0]])
+ else:
+ return None
+
+ # module time cost tags
+ modules_cost_ms = {module: time_cost(cp)
+ for module, cp in module_map.items()}
+
+ tags.update({module: format_tag(cost)
+ for module, cost in modules_cost_ms.items()})
+
+ tags.update({'total': time_cost((check_points[0], check_points[-1]))})
+
+ profile = TEMPLATE.format(**tags)
+
+ logfile = open('{}.json'.format(OUTPUT), 'w')
+ logfile.write(json.dumps(tags))
+
+ print(profile)
+ if log:
+ log.info('%s' % profile)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/doctor_tests/scenario/__init__.py b/doctor_tests/scenario/__init__.py
new file mode 100644
index 00000000..48893ae6
--- /dev/null
+++ b/doctor_tests/scenario/__init__.py
@@ -0,0 +1,8 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
diff --git a/doctor_tests/scenario/common.py b/doctor_tests/scenario/common.py
new file mode 100644
index 00000000..a5cbe483
--- /dev/null
+++ b/doctor_tests/scenario/common.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import sys
+
+from doctor_tests.common.utils import match_rep_in_file
+
+
+def calculate_notification_time():
+ log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log')
+
+ reg = '(?<=doctor monitor detected at )\d+.\d+'
+ result = match_rep_in_file(reg, log_file)
+ if not result:
+ raise Exception('Can not match detected time')
+ detected = result.group(0)
+
+ reg = '(?<=doctor consumer notified at )\d+.\d+'
+ result = match_rep_in_file(reg, log_file)
+ if not result:
+ raise Exception('Can not match notified time')
+ notified = result.group(0)
+
+ return float(notified) - float(detected)
diff --git a/doctor_tests/scenario/network_failure.py b/doctor_tests/scenario/network_failure.py
new file mode 100644
index 00000000..b94a622d
--- /dev/null
+++ b/doctor_tests/scenario/network_failure.py
@@ -0,0 +1,71 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import nova_client
+from doctor_tests.common.utils import SSHClient
+
+LINK_DOWN_SCRIPT = """
+#!/bin/bash -x
+dev=$(sudo ip a | awk '/ {compute_ip}\//{{print $NF}}')
+sleep 1
+sudo ip link set $dev down
+echo "doctor set link down at" $(date "+%s.%N")
+sleep 10
+sudo ip link set $dev up
+sleep 1
+"""
+
+
+class NetworkFault(object):
+
+ def __init__(self, conf, installer, log):
+ self.conf = conf
+ self.log = log
+ self.installer = installer
+ self.nova = nova_client(self.conf.nova_version, get_session())
+ self.host = None
+ self.GetLog = False
+
+ def start(self, host):
+ self.log.info('fault inject start......')
+ self._set_link_down(host.ip)
+ self.host = host
+ self.log.info('fault inject end......')
+
+ def cleanup(self):
+ self.log.info('fault inject cleanup......')
+ self.get_disable_network_log()
+
+ def get_disable_network_log(self):
+ if self.GetLog:
+ self.log.info('Already get the disable_netork.log from down_host......')
+ return
+ if self.host is not None:
+ client = SSHClient(self.host.ip,
+ self.installer.node_user_name,
+ key_filename=self.installer.get_ssh_key_from_installer(),
+ look_for_keys=True,
+ log=self.log)
+ client.scp('disable_network.log', './disable_network.log', method='get')
+ self.log.info('Get the disable_netork.log from down_host(host_name:%s, host_ip:%s)'
+ % (self.host.name, self.host.ip))
+ self.GetLog = True
+
+ def _set_link_down(self, compute_ip):
+ file_name = './disable_network.sh'
+ with open(file_name, 'w') as file:
+ file.write(LINK_DOWN_SCRIPT.format(compute_ip=compute_ip))
+ client = SSHClient(compute_ip,
+ self.installer.node_user_name,
+ key_filename=self.installer.get_ssh_key_from_installer(),
+ look_for_keys=True,
+ log=self.log)
+ client.scp('./disable_network.sh', 'disable_network.sh')
+ command = 'bash disable_network.sh > disable_network.log 2>&1 &'
+ client.ssh(command)
diff --git a/doctor_tests/user.py b/doctor_tests/user.py
new file mode 100644
index 00000000..33f995e7
--- /dev/null
+++ b/doctor_tests/user.py
@@ -0,0 +1,163 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import keystone_client
+from doctor_tests.os_clients import nova_client
+
+
+OPTS = [
+ cfg.StrOpt('doctor_user',
+ default='doctor',
+ help='the name of test user',
+ required=True),
+ cfg.StrOpt('doctor_passwd',
+ default='doctor',
+ help='the password of test user',
+ required=True),
+ cfg.StrOpt('doctor_project',
+ default='doctor',
+ help='the name of test project',
+ required=True),
+ cfg.StrOpt('doctor_role',
+ default='_member_',
+ help='the role of test user',
+ required=True),
+ cfg.IntOpt('quota_instances',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the quota of instances in test user',
+ required=True),
+ cfg.IntOpt('quota_cores',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the quota of cores in test user',
+ required=True),
+]
+
+
+class User(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.keystone = \
+ keystone_client(get_session())
+ self.nova = \
+ nova_client(conf.nova_version, get_session())
+ self.users = {}
+ self.projects = {}
+ self.roles = {}
+ self.roles_for_user = {}
+ self.roles_for_admin = {}
+
+ def create(self):
+ """create test user, project and etc"""
+ self.log.info('user create start......')
+
+ self._create_project()
+ self._create_user()
+ self._create_role()
+ self._add_user_role_in_project(is_admin=False)
+ self._add_user_role_in_project(is_admin=True)
+
+ self.log.info('user create end......')
+
+ def _create_project(self):
+ """create test project"""
+ self.projects = {project.name: project
+ for project in self.keystone.tenants.list()}
+ if self.conf.doctor_project not in self.projects:
+ test_project = \
+ self.keystone.tenants.create(self.conf.doctor_project)
+ self.projects[test_project.name] = test_project
+
+ def _create_user(self):
+ """create test user"""
+ project = self.projects.get(self.conf.doctor_project)
+ self.users = {user.name: user for user in self.keystone.users.list()}
+ if self.conf.doctor_user not in self.users:
+ test_user = self.keystone.users.create(
+ self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ tenant_id=project.id)
+ self.users[test_user.name] = test_user
+
+ def _create_role(self):
+ """create test role"""
+ self.roles = {role.name: role for role in self.keystone.roles.list()}
+ if self.conf.doctor_role not in self.roles:
+ test_role = self.keystone.roles.create(self.conf.doctor_role)
+ self.roles[test_role.name] = test_role
+
+ def _add_user_role_in_project(self, is_admin=False):
+ """add test user with test role in test project"""
+ project = self.projects.get(self.conf.doctor_project)
+
+ user_name = 'admin' if is_admin else self.conf.doctor_user
+ user = self.users.get(user_name)
+
+ role_name = 'admin' if is_admin else self.conf.doctor_role
+ role = self.roles.get(role_name)
+
+ roles_for_user = self.roles_for_admin \
+ if is_admin else self.roles_for_user
+
+ roles_for_user = \
+ {role.name: role for role in
+ self.keystone.roles.roles_for_user(user, tenant=project)}
+ if role_name not in roles_for_user:
+ self.keystone.roles.add_user_role(user, role, tenant=project)
+ roles_for_user[role_name] = role
+
+ def delete(self):
+ """delete the test user, project and role"""
+ self.log.info('user delete start......')
+
+ project = self.projects.get(self.conf.doctor_project)
+ user = self.users.get(self.conf.doctor_user)
+ role = self.roles.get(self.conf.doctor_role)
+
+ if project:
+ if 'admin' in self.roles_for_admin:
+ self.keystone.roles.remove_user_role(
+ self.users['admin'],
+ self.roles['admin'],
+ tenant=project)
+
+ if user:
+ if role and self.conf.doctor_role in self.roles_for_user:
+ self.keystone.roles.remove_user_role(
+ user, role, tenant=project)
+ self.keystone.roles.delete(role)
+ self.keystone.users.delete(user)
+
+ self.keystone.tenants.delete(project)
+ self.log.info('user delete end......')
+
+ def update_quota(self):
+ self.log.info('user quota update start......')
+ project = self.projects.get(self.conf.doctor_project)
+ user = self.users.get(self.conf.doctor_user)
+
+ if project and user:
+ self.quota = self.nova.quotas.get(project.id,
+ user_id=user.id)
+ if self.conf.quota_instances > self.quota.instances:
+ self.nova.quotas.update(project.id,
+ instances=self.conf.quota_instances,
+ user_id=user.id)
+ if self.conf.quota_cores > self.quota.cores:
+ self.nova.quotas.update(project.id,
+ cores=self.conf.quota_cores,
+ user_id=user.id)
+ self.log.info('user quota update end......')
+ else:
+ raise Exception('No project or role for update quota')