diff options
Diffstat (limited to 'doctor_tests')
35 files changed, 2446 insertions, 0 deletions
diff --git a/doctor_tests/alarm.py b/doctor_tests/alarm.py new file mode 100644 index 00000000..3b1aaf3f --- /dev/null +++ b/doctor_tests/alarm.py @@ -0,0 +1,93 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from oslo_config import cfg + +from doctor_tests.identity_auth import get_identity_auth +from doctor_tests.identity_auth import get_session +from doctor_tests.os_clients import aodh_client +from doctor_tests.os_clients import nova_client + +OPTS = [ + cfg.StrOpt('alarm_basename', + default='doctor_alarm', + help='the base name of alarm', + required=True), +] + + +class Alarm(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.auth = get_identity_auth(project=self.conf.doctor_project) + self.aodh = \ + aodh_client(conf.aodh_version, + get_session(auth=self.auth)) + self.nova = \ + nova_client(conf.nova_version, + get_session(auth=self.auth)) + self._init_alarm_name() + + def _init_alarm_name(self): + self.alarm_names = [] + for i in range(0, self.conf.instance_count): + alarm_name = '%s%d' % (self.conf.alarm_basename, i) + self.alarm_names.append(alarm_name) + + def create(self): + self.log.info('alarm create start......') + + alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()} + servers = \ + {getattr(server, 'name'): server + for server in self.nova.servers.list()} + + for i in range(0, self.conf.instance_count): + alarm_name = self.alarm_names[i] + if alarm_name in alarms: + continue; + vm_name = '%s%d' % (self.conf.instance_basename, i) + vm_id = getattr(servers[vm_name], 'id') + alarm_request = dict( + name=alarm_name, + description=u'VM failure', + enabled=True, + alarm_actions=[u'http://%s:%d/failure' + % (self.conf.consumer.ip, + self.conf.consumer.port)], + repeat_actions=False, + severity=u'moderate', + type=u'event', + event_rule=dict( + event_type=u'compute.instance.update', + query=[ + dict(field=u'traits.instance_id', + type='', + op=u'eq', + value=vm_id), + dict(field=u'traits.state', + type='', + op=u'eq', + value=u'error')])) + self.aodh.alarm.create(alarm_request) + + self.log.info('alarm create end......') + + def delete(self): + self.log.info('alarm delete start.......') + + alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()} + for alarm_name in self.alarm_names: + if alarm_name in alarms: + self.aodh.alarm.delete(alarms[alarm_name]['alarm_id']) + + del self.alarm_names[:] + + self.log.info('alarm delete end.......') diff --git a/doctor_tests/common/__init__.py b/doctor_tests/common/__init__.py new file mode 100644 index 00000000..e68a3070 --- /dev/null +++ b/doctor_tests/common/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +##############################################################################
\ No newline at end of file diff --git a/doctor_tests/common/constants.py b/doctor_tests/common/constants.py new file mode 100644 index 00000000..72d037af --- /dev/null +++ b/doctor_tests/common/constants.py @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from collections import namedtuple + + +Host = namedtuple('Host', ['name', 'ip']) diff --git a/doctor_tests/common/utils.py b/doctor_tests/common/utils.py new file mode 100644 index 00000000..2e823acb --- /dev/null +++ b/doctor_tests/common/utils.py @@ -0,0 +1,105 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import json +import os +import paramiko +import re + + +def load_json_file(full_path): + """Loads JSON from file + :param target_filename: + :return: + """ + if not os.path.isfile(full_path): + raise Exception('File(%s) does not exist' % full_path) + + with open(full_path, 'r') as file: + return json.load(file) + + +def write_json_file(full_path, data): + """write JSON from file + :param target_filename: + :return: + """ + + with open(full_path, 'w+') as file: + file.write(json.dumps(data)) + + +def match_rep_in_file(regex, full_path): + if not os.path.isfile(full_path): + raise Exception('File(%s) does not exist' % full_path) + + with open(full_path, 'r') as file: + for line in file: + result = re.search(regex, line) + if result: + return result + + return None + + +class SSHClient(object): + def __init__(self, ip, username, password=None, pkey=None, + key_filename=None, log=None, look_for_keys=False, + allow_agent=False): + self.client = paramiko.SSHClient() + self.client.load_system_host_keys() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.client.connect(ip, username=username, password=password, + pkey=pkey, key_filename=key_filename, + look_for_keys=look_for_keys, + allow_agent=allow_agent) + self.log = log + + def __del__(self): + self.client.close() + + def ssh(self, command): + if self.log: + self.log.info("Executing: %s" % command) + stdin, stdout, stderr = self.client.exec_command(command) + ret = stdout.channel.recv_exit_status() + output = list() + for line in stdout.read().splitlines(): + output.append(line.decode('utf-8')) + if ret: + if self.log: + self.log.info("*** FAILED to run command %s (%s)" % (command, ret)) + raise Exception( + "Unable to run \ncommand: %s\nret: %s" + % (command, ret)) + if self.log: + self.log.info("*** SUCCESSFULLY run command %s" % command) + return ret, output + + def scp(self, source, dest, method='put'): + if self.log: + self.log.info("Copy %s -> %s" % (source, dest)) + ftp = self.client.open_sftp() + if method == 'put': + ftp.put(source, dest) + elif method == 'get': + ftp.get(source, dest) + ftp.close() + + +def run_async(func): + from threading import Thread + from functools import wraps + + @wraps(func) + def async_func(*args, **kwargs): + thread = Thread(target=func, args=args, kwargs=kwargs) + thread.start() + return thread + + return async_func diff --git a/doctor_tests/config.py b/doctor_tests/config.py new file mode 100644 index 00000000..273e84d5 --- /dev/null +++ b/doctor_tests/config.py @@ -0,0 +1,54 @@ +##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import itertools
+
+from oslo_config import cfg
+
+from doctor_tests import alarm
+from doctor_tests import consumer
+from doctor_tests import image
+from doctor_tests import instance
+from doctor_tests import installer
+from doctor_tests import network
+from doctor_tests import inspector
+from doctor_tests import monitor
+from doctor_tests import os_clients
+from doctor_tests import profiler_poc
+from doctor_tests import user
+
+
+def list_opts():
+ return [
+ ('installer', installer.OPTS),
+ ('monitor', monitor.OPTS),
+ ('inspector', inspector.OPTS),
+ ('consumer', consumer.OPTS),
+ ('DEFAULT', itertools.chain(
+ os_clients.OPTS,
+ image.OPTS,
+ user.OPTS,
+ network.OPTS,
+ instance.OPTS,
+ alarm.OPTS,
+ profiler_poc.OPTS))
+ ]
+
+
+def prepare_conf(args=None, conf=None, config_files=None):
+ if conf is None:
+ conf = cfg.ConfigOpts()
+
+ for group, options in list_opts():
+ conf.register_opts(list(options),
+ group=None if group == 'DEFAULT' else group)
+
+ conf(args, project='doctor', validate_default_values=True,
+ default_config_files=config_files)
+
+ return conf
diff --git a/doctor_tests/consumer/__init__.py b/doctor_tests/consumer/__init__.py new file mode 100644 index 00000000..2c66a547 --- /dev/null +++ b/doctor_tests/consumer/__init__.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from oslo_config import cfg +from oslo_utils import importutils + + +OPTS = [ + cfg.StrOpt('type', + default='sample', + choices=['sample'], + help='the component of doctor consumer', + required=True), + cfg.StrOpt('ip', + default='127.0.0.1', + help='the ip of consumer', + required=True), + cfg.IntOpt('port', + default='12346', + help='the port of doctor consumer', + required=True), +] + + +_consumer_name_class_mapping = { + 'sample': 'doctor_tests.consumer.sample.SampleConsumer' +} + + +def get_consumer(conf, log): + consumer_class = _consumer_name_class_mapping.get(conf.consumer.type) + return importutils.import_object(consumer_class, conf, log) diff --git a/doctor_tests/consumer/base.py b/doctor_tests/consumer/base.py new file mode 100644 index 00000000..35170748 --- /dev/null +++ b/doctor_tests/consumer/base.py @@ -0,0 +1,26 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseConsumer(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + + @abc.abstractmethod + def start(self): + pass + + @abc.abstractmethod + def stop(self): + pass
\ No newline at end of file diff --git a/doctor_tests/consumer/sample.py b/doctor_tests/consumer/sample.py new file mode 100644 index 00000000..d76a764b --- /dev/null +++ b/doctor_tests/consumer/sample.py @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from flask import Flask +from flask import request +import json +import time +from threading import Thread +import requests + +from doctor_tests.consumer.base import BaseConsumer + + +class SampleConsumer(BaseConsumer): + + def __init__(self, conf, log): + super(SampleConsumer, self).__init__(conf, log) + self.app = None + + def start(self): + self.log.info('sample consumer start......') + self.app = ConsumerApp(self.conf.consumer.port, self, self.log) + self.app.start() + + def stop(self): + self.log.info('sample consumer stop......') + if not self.app: + return + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + url = 'http://%s:%d/shutdown'\ + % (self.conf.consumer.ip, + self.conf.consumer.port) + requests.post(url, data='', headers=headers) + + +class ConsumerApp(Thread): + + def __init__(self, port, consumer, log): + Thread.__init__(self) + self.port = port + self.consumer = consumer + self.log = log + + def run(self): + app = Flask('consumer') + + @app.route('/failure', methods=['POST']) + def event_posted(): + self.log.info('doctor consumer notified at %s' % time.time()) + self.log.info('sample consumer received data = %s' % request.data) + data = json.loads(request.data.decode('utf8')) + return 'OK' + + @app.route('/shutdown', methods=['POST']) + def shutdown(): + self.log.info('shutdown consumer app server at %s' % time.time()) + func = request.environ.get('werkzeug.server.shutdown') + if func is None: + raise RuntimeError('Not running with the Werkzeug Server') + func() + return 'consumer app shutting down...' + + app.run(host="0.0.0.0", port=self.port) diff --git a/doctor_tests/identity_auth.py b/doctor_tests/identity_auth.py new file mode 100644 index 00000000..2586720c --- /dev/null +++ b/doctor_tests/identity_auth.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2017 NEC Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os + +from keystoneauth1 import loading +from keystoneauth1 import session + + +def get_identity_auth(username=None, password=None, project=None): + auth_url = os.environ['OS_AUTH_URL'] + username = username or os.environ['OS_USERNAME'] + password = password or os.environ['OS_PASSWORD'] + user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default' + user_domain_id = os.environ.get('OS_USER_DOMAIN_ID') or 'default' + project_name = project or os.environ.get('OS_PROJECT_NAME') \ + or os.environ.get('OS_TENANT_NAME') + project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default' + project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID') or 'default' + + loader = loading.get_plugin_loader('password') + return loader.load_from_options( + auth_url=auth_url, + username=username, + password=password, + user_domain_name=user_domain_name, + user_domain_id=user_domain_id, + project_name=project_name, + tenant_name=project_name, + project_domain_name=project_domain_name, + project_domain_id=project_domain_id) + + +def get_session(auth=None): + """Get a user credentials auth session.""" + if auth is None: + auth = get_identity_auth() + return session.Session(auth=auth) diff --git a/doctor_tests/image.py b/doctor_tests/image.py new file mode 100644 index 00000000..2e313e12 --- /dev/null +++ b/doctor_tests/image.py @@ -0,0 +1,74 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import urllib.request + +from oslo_config import cfg + +from doctor_tests.identity_auth import get_session +from doctor_tests.os_clients import glance_client + +OPTS = [ + cfg.StrOpt('image_name', + default=os.environ.get('IMAGE_NAME', 'cirros'), + help='the name of test image', + required=True), + cfg.StrOpt('image_format', + default='qcow2', + help='the format of test image', + required=True), + cfg.StrOpt('image_filename', + default='cirros.img', + help='the name of image file', + required=True), + cfg.StrOpt('image_download_url', + default='https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img', + help='the url where to get the image', + required=True), +] + + +class Image(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.glance = \ + glance_client(conf.glance_version, get_session()) + self.use_existing_image = False + self.image = None + + def create(self): + self.log.info('image create start......') + + images = {image.name: image for image in self.glance.images.list()} + if self.conf.image_name not in images: + if not os.path.exists(self.conf.image_filename): + resp = urllib.request.urlopen(self.conf.image_download_url) + with open(self.conf.image_filename, "wb") as file: + file.write(resp.read()) + self.image = self.glance.images.create(name=self.conf.image_name, + disk_format=self.conf.image_format, + container_format="bare", + visibility="public") + self.glance.images.upload(self.image['id'], + open(self.conf.image_filename, 'rb')) + else: + self.use_existing_image = True + self.image = images[self.conf.image_name] + + self.log.info('image create end......') + + def delete(self): + self.log.info('image delete start.......') + + if not self.use_existing_image and self.image: + self.glance.images.delete(self.image['id']) + + self.log.info('image delete end.......') diff --git a/doctor_tests/inspector/__init__.py b/doctor_tests/inspector/__init__.py new file mode 100644 index 00000000..3be79e57 --- /dev/null +++ b/doctor_tests/inspector/__init__.py @@ -0,0 +1,40 @@ +############################################################################# +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os + +from oslo_config import cfg +from oslo_utils import importutils + + +OPTS = [ + cfg.StrOpt('type', + default=os.environ.get('INSPECTOR_TYPE', 'sample'), + choices=['sample', 'congress', 'vitrage'], + help='the component of doctor inspector', + required=True), + cfg.StrOpt('ip', + default='127.0.0.1', + help='the host ip of inspector', + required=False), + cfg.StrOpt('port', + default='12345', + help='the port of default for inspector', + required=False), +] + + +_inspector_name_class_mapping = { + 'sample': 'doctor_tests.inspector.sample.SampleInspector', + 'congress': 'doctor_tests.inspector.congress.CongressInspector', +} + + +def get_inspector(conf, log): + inspector_class = _inspector_name_class_mapping[conf.inspector.type] + return importutils.import_object(inspector_class, conf, log) diff --git a/doctor_tests/inspector/base.py b/doctor_tests/inspector/base.py new file mode 100644 index 00000000..854f0695 --- /dev/null +++ b/doctor_tests/inspector/base.py @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseInspector(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + + @abc.abstractmethod + def get_inspector_url(self): + pass + + @abc.abstractmethod + def start(self): + pass + + @abc.abstractmethod + def stop(self): + pass
\ No newline at end of file diff --git a/doctor_tests/inspector/congress.py b/doctor_tests/inspector/congress.py new file mode 100644 index 00000000..c89a41bd --- /dev/null +++ b/doctor_tests/inspector/congress.py @@ -0,0 +1,94 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from doctor_tests.identity_auth import get_identity_auth +from doctor_tests.identity_auth import get_session +from doctor_tests.os_clients import congress_client + +from doctor_tests.inspector.base import BaseInspector + + +class CongressInspector(BaseInspector): + nova_api_min_version = '2.11' + doctor_driver = 'doctor' + doctor_datasource = 'doctor' + policy = 'classification' + rules = { + 'host_down': + 'host_down(host) :- doctor:events(hostname=host, type="compute.host.down", status="down")', + 'active_instance_in_host': + 'active_instance_in_host(vmid, host) :- nova:servers(id=vmid, host_name=host, status="ACTIVE")', + 'host_force_down': + 'execute[nova:services.force_down(host, "nova-compute", "True")] :- host_down(host)', + 'error_vm_states': + 'execute[nova:servers.reset_state(vmid, "error")] :- host_down(host), active_instance_in_host(vmid, host)' + } + + def __init__(self, conf, log): + super(CongressInspector, self).__init__(conf, log) + self.auth = get_identity_auth() + self.congress = congress_client(get_session(auth=self.auth)) + self._init_driver_and_ds() + self.inspector_url = self.get_inspector_url() + + def _init_driver_and_ds(self): + datasources = \ + {ds['name']: ds for ds in self.congress.list_datasources()['results']} + + # check nova_api version + nova_api_version = datasources['nova']['config'].get('api_version') + if nova_api_version and nova_api_version < self.nova_api_min_version: + raise Exception('Congress Nova datasource API version < nova_api_min_version(%s)' + % self.nova_api_min_version) + + # create doctor datasource if it's not exist + if self.doctor_datasource not in datasources: + self.congress.create_datasource( + body={'driver': self.doctor_driver, + 'name': self.doctor_datasource}) + + # check whether doctor driver exist + drivers = \ + {driver['id']: driver for driver in self.congress.list_drivers()['results']} + if self.doctor_driver not in drivers: + raise Exception('Do not support doctor driver in congress') + + self.policy_rules = \ + {rule['name']: rule for rule in + self.congress.list_policy_rules(self.policy)['results']} + + def get_inspector_url(self): + ds = self.congress.list_datasources()['results'] + doctor_ds = next((item for item in ds if item['driver'] == 'doctor'), + None) + congress_endpoint = self.congress.httpclient.get_endpoint(auth=self.auth) + return ('%s/v1/data-sources/%s/tables/events/rows' % + (congress_endpoint, doctor_ds['id'])) + + def start(self): + self.log.info('congress inspector start......') + + for rule_name, rule in self.rules.items(): + self._add_rule(rule_name, rule) + + def stop(self): + self.log.info('congress inspector stop......') + + for rule_name in self.rules.keys(): + self._del_rule(rule_name) + + def _add_rule(self, rule_name, rule): + if rule_name not in self.policy_rules: + self.congress.create_policy_rule(self.policy, + body={'name': rule_name, + 'rule': rule}) + + def _del_rule(self, rule_name): + if rule_name in self.policy_rules: + rule_id = self.policy_rules[rule_name]['id'] + self.congress.delete_policy_rule(self.policy, rule_id) diff --git a/doctor_tests/inspector/sample.py b/doctor_tests/inspector/sample.py new file mode 100644 index 00000000..114e4ebd --- /dev/null +++ b/doctor_tests/inspector/sample.py @@ -0,0 +1,169 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import collections +from flask import Flask +from flask import request +import json +import time +from threading import Thread +import requests + +from doctor_tests.common import utils +from doctor_tests.identity_auth import get_identity_auth +from doctor_tests.identity_auth import get_session +from doctor_tests.os_clients import nova_client +from doctor_tests.os_clients import neutron_client +from doctor_tests.inspector.base import BaseInspector + + +class SampleInspector(BaseInspector): + event_type = 'compute.host.down' + + def __init__(self, conf, log): + super(SampleInspector, self).__init__(conf, log) + self.inspector_url = self.get_inspector_url() + self.novaclients = list() + self._init_novaclients() + # Normally we use this client for non redundant API calls + self.nova = self.novaclients[0] + + auth = get_identity_auth(project=self.conf.doctor_project) + session = get_session(auth=auth) + self.neutron = neutron_client(session) + + self.servers = collections.defaultdict(list) + self.hostnames = list() + self.app = None + + def _init_novaclients(self): + self.NUMBER_OF_CLIENTS = self.conf.instance_count + auth = get_identity_auth(project=self.conf.doctor_project) + session = get_session(auth=auth) + for i in range(self.NUMBER_OF_CLIENTS): + self.novaclients.append( + nova_client(self.conf.nova_version, session)) + + def _init_servers_list(self): + self.servers.clear() + opts = {'all_tenants': True} + servers = self.nova.servers.list(search_opts=opts) + for server in servers: + try: + host = server.__dict__.get('OS-EXT-SRV-ATTR:host') + self.servers[host].append(server) + self.log.debug('get hostname=%s from server=%s' % (host, server)) + except Exception as e: + self.log.info('can not get hostname from server=%s' % server) + + def get_inspector_url(self): + return 'http://%s:%s' % (self.conf.inspector.ip, self.conf.inspector.port) + + def start(self): + self.log.info('sample inspector start......') + self._init_servers_list() + self.app = InspectorApp(self.conf.inspector.port, self, self.log) + self.app.start() + + def stop(self): + self.log.info('sample inspector stop......') + if not self.app: + return + for hostname in self.hostnames: + self.nova.services.force_down(hostname, 'nova-compute', False) + + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + url = '%s%s' % (self.inspector_url, 'shutdown') \ + if self.inspector_url.endswith('/') else \ + '%s%s' % (self.inspector_url, '/shutdown') + requests.post(url, data='', headers=headers) + + def handle_events(self, events): + for event in events: + hostname = event['details']['hostname'] + event_type = event['type'] + if event_type == self.event_type: + self.hostnames.append(hostname) + thr1 = self._disable_compute_host(hostname) + thr2 = self._vms_reset_state('error', hostname) + thr3 = self._set_ports_data_plane_status('DOWN', hostname) + thr1.join() + thr2.join() + thr3.join() + + @utils.run_async + def _disable_compute_host(self, hostname): + self.nova.services.force_down(hostname, 'nova-compute', True) + self.log.info('doctor mark host(%s) down at %s' % (hostname, time.time())) + + @utils.run_async + def _vms_reset_state(self, state, hostname): + + @utils.run_async + def _vm_reset_state(nova, server, state): + nova.servers.reset_state(server, state) + self.log.info('doctor mark vm(%s) error at %s' % (server, time.time())) + + thrs = [] + for nova, server in zip(self.novaclients, self.servers[hostname]): + t = _vm_reset_state(nova, server, state) + thrs.append(t) + for t in thrs: + t.join() + + @utils.run_async + def _set_ports_data_plane_status(self, status, hostname): + body = {'data_plane_status': status} + + @utils.run_async + def _set_port_data_plane_status(port_id): + self.neutron.update_port(port_id, body) + self.log.info('doctor set data plane status %s on port %s' % (status, port_id)) + + thrs = [] + params = {'binding:host_id': hostname} + for port_id in self.neutron.list_ports(**params): + t = _set_port_data_plane_status(port_id) + thrs.append(t) + for t in thrs: + t.join() + + +class InspectorApp(Thread): + + def __init__(self, port, inspector, log): + Thread.__init__(self) + self.port = port + self.inspector = inspector + self.log = log + + def run(self): + app = Flask('inspector') + + @app.route('/events', methods=['PUT']) + def event_posted(): + self.log.info('event posted in sample inspector at %s' % time.time()) + self.log.info('sample inspector = %s' % self.inspector) + self.log.info('sample inspector received data = %s' % request.data) + events = json.loads(request.data.decode('utf8')) + self.inspector.handle_events(events) + return "OK" + + @app.route('/shutdown', methods=['POST']) + def shutdown(): + self.log.info('shutdown inspector app server at %s' % time.time()) + func = request.environ.get('werkzeug.server.shutdown') + if func is None: + raise RuntimeError('Not running with the Werkzeug Server') + func() + return 'inspector app shutting down...' + + app.run(host="0.0.0.0", port=self.port) diff --git a/doctor_tests/installer/__init__.py b/doctor_tests/installer/__init__.py new file mode 100644 index 00000000..02735b11 --- /dev/null +++ b/doctor_tests/installer/__init__.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os + +from oslo_config import cfg +from oslo_utils import importutils + +OPTS = [ + cfg.StrOpt('type', + default=os.environ.get('INSTALLER_TYPE', 'local'), + choices=['local', 'apex'], + help='the type of installer', + required=True), + cfg.StrOpt('ip', + default=os.environ.get('INSTALLER_IP', '127.0.0.1'), + help='the ip of installer'), + cfg.StrOpt('username', + default='root', + help='the user name for login installer server', + required=True), +] + + +_installer_name_class_mapping = { + 'local': 'doctor_tests.installer.local.LocalInstaller', + 'apex': 'doctor_tests.installer.apex.ApexInstaller' +} + + +def get_installer(conf, log): + installer_class = _installer_name_class_mapping[conf.installer.type] + return importutils.import_object(installer_class, conf, log) diff --git a/doctor_tests/installer/apex.py b/doctor_tests/installer/apex.py new file mode 100644 index 00000000..2a1ce94b --- /dev/null +++ b/doctor_tests/installer/apex.py @@ -0,0 +1,126 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import getpass +import grp +import os +import pwd +import stat +import subprocess + +from doctor_tests.common.utils import SSHClient +from doctor_tests.installer.base import BaseInstaller + + +class ApexInstaller(BaseInstaller): + node_user_name = 'heat-admin' + cm_set_script = 'set_ceilometer.py' + cm_restore_script = 'restore_ceilometer.py' + + def __init__(self, conf, log): + super(ApexInstaller, self).__init__(conf, log) + self.client = SSHClient(self.conf.installer.ip, + self.conf.installer.username, + look_for_keys=True) + self.key_file = None + self.controllers = list() + self.controller_clients = list() + self.servers = list() + + def setup(self): + self.log.info('Setup Apex installer start......') + + self.get_ssh_key_from_installer() + self.get_controller_ips() + self.set_apply_patches() + self.setup_stunnel() + + def cleanup(self): + self.restore_apply_patches() + for server in self.servers: + server.terminate() + + def get_ssh_key_from_installer(self): + self.log.info('Get SSH keys from Apex installer......') + + if self.key_file is not None: + self.log.info('Already have SSH keys from Apex installer......') + return self.key_file + + self.client.scp('/home/stack/.ssh/id_rsa', './instack_key', method='get') + user = getpass.getuser() + uid = pwd.getpwnam(user).pw_uid + gid = grp.getgrnam(user).gr_gid + os.chown('./instack_key', uid, gid) + os.chmod('./instack_key', stat.S_IREAD) + current_dir = os.curdir + self.key_file = '{0}/{1}'.format(current_dir, 'instack_key') + return self.key_file + + def get_controller_ips(self): + self.log.info('Get controller ips from Apex installer......') + + command = "source stackrc; " \ + "nova list | grep ' overcloud-controller-[0-9] ' " \ + "| sed -e 's/^.*ctlplane=//' |awk '{print $1}'" + ret, controllers = self.client.ssh(command) + if ret: + raise Exception('Exec command to get controller ips in Apex installer failed' + 'ret=%s, output=%s' % (ret, controllers)) + self.log.info('Get controller_ips:%s from Apex installer' % controllers) + self.controllers = controllers + + def get_host_ip_from_hostname(self, hostname): + self.log.info('Get host ip from host name in Apex installer......') + + hostname_in_undercloud = hostname.split('.')[0] + + command = "source stackrc; nova show %s | awk '/ ctlplane network /{print $5}'" % (hostname_in_undercloud) + ret, host_ip = self.client.ssh(command) + if ret: + raise Exception('Exec command to get host ip from hostname(%s) in Apex installer failed' + 'ret=%s, output=%s' % (hostname, ret, host_ip)) + self.log.info('Get host_ip:%s from host_name:%s in Apex installer' % (host_ip, hostname)) + return host_ip[0] + + def setup_stunnel(self): + self.log.info('Setup ssh stunnel in controller nodes in Apex installer......') + for node_ip in self.controllers: + cmd = "sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s -R %s:localhost:%s sleep 600 > ssh_tunnel.%s.log 2>&1 < /dev/null &" \ + % (self.key_file, self.node_user_name, node_ip, + self.conf.consumer.port, self.conf.consumer.port, node_ip) + server = subprocess.Popen(cmd, shell=True) + self.servers.append(server) + server.communicate() + + def set_apply_patches(self): + self.log.info('Set apply patches start......') + + for node_ip in self.controllers: + client = SSHClient(node_ip, self.node_user_name, key_filename=self.key_file) + self.controller_clients.append(client) + self._ceilometer_apply_patches(client, self.cm_set_script) + + def restore_apply_patches(self): + self.log.info('restore apply patches start......') + + for client in self.controller_clients: + self._ceilometer_apply_patches(client, self.cm_restore_script) + + def _ceilometer_apply_patches(self, ssh_client, script_name): + installer_dir = os.path.dirname(os.path.realpath(__file__)) + script_abs_path = '{0}/{1}/{2}'.format(installer_dir, 'common', script_name) + + ssh_client.scp(script_abs_path, script_name) + cmd = 'sudo python %s' % script_name + ret, output = ssh_client.ssh(cmd) + if ret: + raise Exception('Do the ceilometer command in controller node failed....' + 'ret=%s, cmd=%s, output=%s' % (ret, cmd, output)) + ssh_client.ssh('sudo systemctl restart openstack-ceilometer-notification.service') + diff --git a/doctor_tests/installer/base.py b/doctor_tests/installer/base.py new file mode 100644 index 00000000..fa39816a --- /dev/null +++ b/doctor_tests/installer/base.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import abc +import six + +@six.add_metaclass(abc.ABCMeta) +class BaseInstaller(object): + def __init__(self, conf, log): + self.conf = conf + self.log = log + + @abc.abstractproperty + def node_user_name(self): + """user name for login to cloud node""" + + @abc.abstractmethod + def get_ssh_key_from_installer(self): + pass + + @abc.abstractmethod + def get_host_ip_from_hostname(self, hostname): + pass + + @abc.abstractmethod + def setup(self): + pass + + @abc.abstractmethod + def cleanup(self): + pass diff --git a/doctor_tests/installer/common/congress.py b/doctor_tests/installer/common/congress.py new file mode 100644 index 00000000..db882de2 --- /dev/null +++ b/doctor_tests/installer/common/congress.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +def set_doctor_driver_conf(ssh_client, restart_cmd): + cg_set_cmd = '''#!/bin/bash +co_conf=/etc/congress/congress.conf +co_conf_bak=/etc/congress/congress.conf.bak +co_entry="congress.datasources.doctor_driver.DoctorDriver" +if sudo grep -e "^drivers.*$co_entry" $co_conf; then + echo "NOTE: congress is configured as we needed" +else + echo "modify the congress config" + sudo cp $co_conf $co_conf_bak + sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf + %s +fi + ''' % (restart_cmd) + + ret, output = ssh_client.ssh(cg_set_cmd) + if ret: + raise Exception('Do the congress command in controller node failed....' + 'ret=%s, cmd=%s, output=%s' % (ret, cg_set_cmd, output)) + + +def restore_doctor_driver_conf(ssh_client, restart_cmd): + cg_restore_cmd = '''#!/bin/bash +co_conf=/etc/congress/congress.conf +co_conf_bak=/etc/congress/congress.conf.bak +if [ -e $co_conf_bak ]; then + echo "restore the congress config" + sudo cp $co_conf_bak $co_conf + sudo rm $co_conf_bak + %s +else + echo "Do not need to restore the congress config" +fi + ''' % (restart_cmd) + + ret, output = ssh_client.ssh(cg_restore_cmd) + if ret: + raise Exception('Do the congress command in controller node failed....' + 'ret=%s, cmd=%s, output=%s' % (ret, cg_restore_cmd, output)) diff --git a/doctor_tests/installer/common/restore_ceilometer.py b/doctor_tests/installer/common/restore_ceilometer.py new file mode 100644 index 00000000..d25b9ede --- /dev/null +++ b/doctor_tests/installer/common/restore_ceilometer.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import shutil + +ep_file = '/etc/ceilometer/event_pipeline.yaml' +ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak' + + +def restore_ep_config(): + + if not os.path.isfile(ep_file_bak): + print('Bak_file:%s does not exist.' % ep_file_bak) + else: + print('restore') + shutil.copyfile(ep_file_bak, ep_file) + os.remove(ep_file_bak) + return + + +restore_ep_config() diff --git a/doctor_tests/installer/common/set_ceilometer.py b/doctor_tests/installer/common/set_ceilometer.py new file mode 100644 index 00000000..f5946cb2 --- /dev/null +++ b/doctor_tests/installer/common/set_ceilometer.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import shutil +import yaml + +ep_file = '/etc/ceilometer/event_pipeline.yaml' +ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak' +event_notifier_topic = 'notifier://?topic=alarm.all' + + +def set_notifier_topic(): + config_modified = False + + if not os.path.isfile(ep_file): + raise Exception("File doesn't exist: %s." % ep_file) + + with open(ep_file, 'r') as file: + config = yaml.safe_load(file) + + sinks = config['sinks'] + for sink in sinks: + if sink['name'] == 'event_sink': + publishers = sink['publishers'] + if event_notifier_topic not in publishers: + print('Add event notifier in ceilometer') + publishers.append(event_notifier_topic) + config_modified = True + else: + print('NOTE: event notifier is configured in ceilometer as we needed') + + if config_modified: + shutil.copyfile(ep_file, ep_file_bak) + with open(ep_file, 'w+') as file: + file.write(yaml.safe_dump(config)) + + +set_notifier_topic() diff --git a/doctor_tests/installer/local.py b/doctor_tests/installer/local.py new file mode 100644 index 00000000..7d0ae542 --- /dev/null +++ b/doctor_tests/installer/local.py @@ -0,0 +1,109 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import shutil +import subprocess + +from doctor_tests.installer.base import BaseInstaller +from doctor_tests.common.utils import load_json_file +from doctor_tests.common.utils import write_json_file + + +class LocalInstaller(BaseInstaller): + node_user_name = 'root' + + nova_policy_file = '/etc/nova/policy.json' + nova_policy_file_backup = '%s%s' % (nova_policy_file, '.bak') + + def __init__(self, conf, log): + super(LocalInstaller, self).__init__(conf, log) + self.policy_modified = False + self.add_policy_file = False + + def setup(self): + self.get_ssh_key_from_installer() + self.set_apply_patches() + + def cleanup(self): + self.restore_apply_patches() + + def get_ssh_key_from_installer(self): + self.log.info('Assuming SSH keys already exchanged with computer for local installer type') + return None + + def get_host_ip_from_hostname(self, hostname): + self.log.info('Get host ip from host name in local installer......') + + cmd = "getent hosts %s | awk '{ print $1 }'" % (hostname) + server = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + stdout, stderr = server.communicate() + host_ip = stdout.strip() + + self.log.info('Get host_ip:%s from host_name:%s in local installer' % (host_ip, hostname)) + return host_ip + + def set_apply_patches(self): + self._set_nova_policy() + + def restore_apply_patches(self): + self._restore_nova_policy() + + def _set_nova_policy(self): + host_status_policy = 'os_compute_api:servers:show:host_status' + host_status_rule = 'rule:admin_or_owner' + policy_data = { + 'context_is_admin': 'role:admin', + 'owner': 'user_id:%(user_id)s', + 'admin_or_owner': 'rule:context_is_admin or rule:owner', + host_status_policy: host_status_rule + } + + if os.path.isfile(self.nova_policy_file): + data = load_json_file(self.nova_policy_file) + if host_status_policy in data: + rule_origion = data[host_status_policy] + if host_status_rule == rule_origion: + self.log.info('Do not need to modify nova policy.') + self.policy_modified = False + else: + # update the host_status_policy + data[host_status_policy] = host_status_rule + self.policy_modified = True + else: + # add the host_status_policy, if the admin_or_owner is not + # defined, add it also + for policy, rule in policy_data.items(): + if policy not in data: + data[policy] = rule + self.policy_modified = True + if self.policy_modified: + self.log.info('Nova policy is Modified.') + shutil.copyfile(self.nova_policy_file, + self.nova_policy_file_backup) + else: + # file does not exit, create a new one and add the policy + self.log.info('Nova policy file not exist. Creating a new one') + data = policy_data + self.add_policy_file = True + + if self.policy_modified or self.add_policy_file: + write_json_file(self.nova_policy_file, data) + os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"') + + def _restore_nova_policy(self): + if self.policy_modified: + shutil.copyfile(self.nova_policy_file_backup, self.nova_policy_file) + os.remove(self.nova_policy_file_backup) + elif self.add_policy_file: + os.remove(self.nova_policy_file) + + if self.add_policy_file or self.policy_modified: + os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"') + self.add_policy_file = False + self.policy_modified = False diff --git a/doctor_tests/instance.py b/doctor_tests/instance.py new file mode 100644 index 00000000..27f412e2 --- /dev/null +++ b/doctor_tests/instance.py @@ -0,0 +1,114 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import time + +from oslo_config import cfg + +from doctor_tests.identity_auth import get_identity_auth +from doctor_tests.identity_auth import get_session +from doctor_tests.os_clients import neutron_client +from doctor_tests.os_clients import nova_client + +OPTS = [ + cfg.StrOpt('flavor', + default='m1.tiny', + help='the name of flavor', + required=True), + cfg.IntOpt('instance_count', + default=os.environ.get('VM_COUNT', 1), + help='the count of instance', + required=True), + cfg.StrOpt('instance_basename', + default='doctor_vm', + help='the base name of instance', + required=True), +] + + +class Instance(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.auth = get_identity_auth(username=self.conf.doctor_user, + password=self.conf.doctor_passwd, + project=self.conf.doctor_project) + self.nova = \ + nova_client(conf.nova_version, + get_session(auth=self.auth)) + self.neutron = neutron_client(get_session(auth=self.auth)) + self.servers = {} + self.vm_names = [] + + def create(self): + self.log.info('instance create start......') + + # get flavor, image and network for vm boot + flavors = {flavor.name: flavor for flavor in self.nova.flavors.list()} + flavor = flavors.get(self.conf.flavor) + image = self.nova.glance.find_image(self.conf.image_name) + network = self.neutron.list_networks(name=self.conf.net_name)['networks'][0] + nics = {'net-id': network['id']} + + self.servers = \ + {getattr(server, 'name'): server + for server in self.nova.servers.list()} + for i in range(0, self.conf.instance_count): + vm_name = "%s%d"%(self.conf.instance_basename, i) + self.vm_names.append(vm_name) + if vm_name not in self.servers: + server = self.nova.servers.create(vm_name, image, + flavor, nics=[nics]) + self.servers[vm_name] = server + time.sleep(0.1) + + self.log.info('instance create end......') + + def delete(self): + self.log.info('instance delete start.......') + + for vm_name in self.vm_names: + if vm_name in self.servers: + self.nova.servers.delete(self.servers[vm_name]) + time.sleep(0.1) + + # check that all vms are deleted + while self.nova.servers.list(): + time.sleep(0.1) + self.servers.clear() + del self.vm_names[:] + + self.log.info('instance delete end.......') + + def wait_for_vm_launch(self): + self.log.info('wait for vm launch start......') + + wait_time = 60 + count = 0 + while count < wait_time: + active_count = 0 + for vm_name in self.vm_names: + server = self.nova.servers.get(self.servers[vm_name]) + server_status = getattr(server, 'status').lower() + if 'active' == server_status: + active_count += 1 + elif 'error' == server_status: + raise Exception('vm launched with error state') + else: + time.sleep(2) + count += 1 + continue + if active_count == self.conf.instance_count: + self.log.info('wait for vm launch end......') + return + count += 1 + time.sleep(2) + raise Exception('time out for vm launch') + diff --git a/doctor_tests/logger.py b/doctor_tests/logger.py new file mode 100644 index 00000000..b7a49fdb --- /dev/null +++ b/doctor_tests/logger.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# Usage: +# import doctor_logger +# logger = doctor_logger.Logger("script_name").getLogger() +# logger.info("message to be shown with - INFO - ") +# logger.debug("message to be shown with - DEBUG -") + +import logging +import os + + +class Logger(object): + def __init__(self, logger_name): + + CI_DEBUG = os.getenv('CI_DEBUG') + + self.logger = logging.getLogger(logger_name) + self.logger.propagate = 0 + self.logger.setLevel(logging.DEBUG) + + formatter = logging.Formatter('%(asctime)s %(filename)s %(lineno)d ' + '%(levelname)-6s %(message)s') + + ch = logging.StreamHandler() + ch.setFormatter(formatter) + if CI_DEBUG is not None and CI_DEBUG.lower() == "true": + ch.setLevel(logging.DEBUG) + else: + ch.setLevel(logging.INFO) + self.logger.addHandler(ch) + + filename = '%s.log' % logger_name + file_handler = logging.FileHandler(filename, mode='w') + file_handler.setFormatter(formatter) + file_handler.setLevel(logging.DEBUG) + self.logger.addHandler(file_handler) + + def getLogger(self): + return self.logger diff --git a/doctor_tests/main.py b/doctor_tests/main.py new file mode 100644 index 00000000..006aac9f --- /dev/null +++ b/doctor_tests/main.py @@ -0,0 +1,215 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +from os.path import isfile, join +import random +import sys +import time + +from doctor_tests.alarm import Alarm +from doctor_tests.common.constants import Host +from doctor_tests.common.utils import match_rep_in_file +from doctor_tests import config +from doctor_tests.consumer import get_consumer +from doctor_tests.identity_auth import get_identity_auth +from doctor_tests.identity_auth import get_session +from doctor_tests.image import Image +from doctor_tests.instance import Instance +from doctor_tests.inspector import get_inspector +from doctor_tests.installer import get_installer +import doctor_tests.logger as doctor_log +from doctor_tests.network import Network +from doctor_tests.monitor import get_monitor +from doctor_tests.os_clients import nova_client +from doctor_tests.profiler_poc import main as profiler_main +from doctor_tests.scenario.common import calculate_notification_time +from doctor_tests.scenario.network_failure import NetworkFault +from doctor_tests.user import User + + +LOG = doctor_log.Logger('doctor').getLogger() + + +class DoctorTest(object): + + def __init__(self, conf): + self.conf = conf + self.image = Image(self.conf, LOG) + self.user = User(self.conf, LOG) + self.network = Network(self.conf, LOG) + self.instance = Instance(self.conf, LOG) + self.alarm = Alarm(self.conf, LOG) + self.installer = get_installer(self.conf, LOG) + self.inspector = get_inspector(self.conf, LOG) + self.monitor = get_monitor(self.conf, + self.inspector.get_inspector_url(), + LOG) + self.consumer = get_consumer(self.conf, LOG) + self.fault = NetworkFault(self.conf, self.installer, LOG) + auth = get_identity_auth(project=self.conf.doctor_project) + self.nova = nova_client(self.conf.nova_version, + get_session(auth=auth)) + self.down_host = None + + def setup(self): + # prepare the cloud env + self.installer.setup() + + # preparing VM image... + self.image.create() + + # creating test user... + self.user.create() + self.user.update_quota() + + # creating VM... + self.network.create() + self.instance.create() + self.instance.wait_for_vm_launch() + + # creating alarm... + self.alarm.create() + + # starting doctor sample components... + self.inspector.start() + + self.down_host = self.get_host_info_for_random_vm() + self.monitor.start(self.down_host) + + self.consumer.start() + + def run(self): + """run doctor test""" + try: + LOG.info('doctor test starting.......') + + # prepare test env + self.setup() + + # wait for aodh alarms are updated in caches for event evaluator, + # sleep time should be larger than event_alarm_cache_ttl(default 60) + time.sleep(60) + + # injecting host failure... + # NOTE (umar) add INTERFACE_NAME logic to host injection + + self.fault.start(self.down_host) + time.sleep(10) + + # verify the test results + # NOTE (umar) copy remote monitor.log file when monitor=collectd + self.check_host_status(self.down_host.name, 'down') + + notification_time = calculate_notification_time() + if notification_time < 1 and notification_time > 0: + LOG.info('doctor test successfully, notification_time=%s' % notification_time) + else: + LOG.error('doctor test failed, notification_time=%s' % notification_time) + sys.exit(1) + + if self.conf.profiler_type: + LOG.info('doctor test begin to run profile.......') + self.collect_logs() + self.run_profiler() + except Exception as e: + LOG.error('doctor test failed, Exception=%s' % e) + sys.exit(1) + finally: + self.cleanup() + + def get_host_info_for_random_vm(self): + num = random.randint(0, self.conf.instance_count - 1) + vm_name = "%s%d" % (self.conf.instance_basename, num) + + servers = \ + {getattr(server, 'name'): server + for server in self.nova.servers.list()} + server = servers.get(vm_name) + if not server: + raise \ + Exception('Can not find instance: vm_name(%s)' % vm_name) + host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname') + host_ip = self.installer.get_host_ip_from_hostname(host_name) + + LOG.info('Get host info(name:%s, ip:%s) which vm(%s) launched at' + % (host_name, host_ip, vm_name)) + return Host(host_name, host_ip) + + def check_host_status(self, hostname, state): + service = self.nova.services.list(host=hostname, binary='nova-compute') + host_state = service[0].__dict__.get('state') + assert host_state == state + + def unset_forced_down_hosts(self): + if self.down_host: + self.nova.services.force_down(self.down_host.name, 'nova-compute', False) + time.sleep(2) + self.check_host_status(self.down_host.name, 'up') + + def collect_logs(self): + self.fault.get_disable_network_log() + + def run_profiler(self): + + log_file = '{0}/{1}'.format(sys.path[0], 'disable_network.log') + reg = '(?<=doctor set link down at )\d+.\d+' + linkdown = float(match_rep_in_file(reg, log_file).group(0)) + + log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log') + reg = '(.* doctor mark vm.* error at )(\d+.\d+)' + vmdown = float(match_rep_in_file(reg, log_file).group(2)) + + reg = '(?<=doctor mark host.* down at )\d+.\d+' + hostdown = float(match_rep_in_file(reg, log_file).group(2)) + + reg = '(?<=doctor monitor detected at )\d+.\d+' + detected = float(match_rep_in_file(reg, log_file).group(0)) + + reg = '(?<=doctor consumer notified at )\d+.\d+' + notified = float(match_rep_in_file(reg, log_file).group(0)) + + # TODO(yujunz) check the actual delay to verify time sync status + # expected ~1s delay from $trigger to $linkdown + relative_start = linkdown + os.environ['DOCTOR_PROFILER_T00'] = str(int((linkdown - relative_start)*1000)) + os.environ['DOCTOR_PROFILER_T01'] = str(int((detected - relative_start) * 1000)) + os.environ['DOCTOR_PROFILER_T03'] = str(int((vmdown - relative_start) * 1000)) + os.environ['DOCTOR_PROFILER_T04'] = str(int((hostdown - relative_start) * 1000)) + os.environ['DOCTOR_PROFILER_T09'] = str(int((notified - relative_start) * 1000)) + + profiler_main(log=LOG) + + def cleanup(self): + self.unset_forced_down_hosts() + self.inspector.stop() + self.monitor.stop() + self.consumer.stop() + self.installer.cleanup() + self.alarm.delete() + self.instance.delete() + self.network.delete() + self.image.delete() + self.fault.cleanup() + self.user.delete() + + +def main(): + """doctor main""" + test_dir = os.path.split(os.path.realpath(__file__))[0] + doctor_root_dir = os.path.dirname(test_dir) + + config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/') + config_files = [join(config_file_dir, f) for f in os.listdir(config_file_dir) + if isfile(join(config_file_dir, f))] + + conf = config.prepare_conf(args=sys.argv[1:], + config_files=config_files) + + doctor = DoctorTest(conf) + doctor.run() diff --git a/doctor_tests/monitor/__init__.py b/doctor_tests/monitor/__init__.py new file mode 100644 index 00000000..7e30c9f8 --- /dev/null +++ b/doctor_tests/monitor/__init__.py @@ -0,0 +1,29 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from oslo_config import cfg +from oslo_utils import importutils + +OPTS = [ + cfg.StrOpt('type', + default='sample', + choices=['sample', 'collectd'], + help='the type of doctor monitor component', + required=True), +] + + +_monitor_name_class_mapping = { + 'sample': 'doctor_tests.monitor.sample.SampleMonitor', + 'collectd': 'doctor_tests.monitor.collectd.CollectdMonitor' +} + +def get_monitor(conf, inspector_url, log): + monitor_class = _monitor_name_class_mapping.get(conf.monitor.type) + return importutils.import_object(monitor_class, conf, + inspector_url, log) diff --git a/doctor_tests/monitor/base.py b/doctor_tests/monitor/base.py new file mode 100644 index 00000000..119c8a1c --- /dev/null +++ b/doctor_tests/monitor/base.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseMonitor(object): + """Monitor computer fault and report error to the inspector""" + def __init__(self, conf, inspector_url, log): + self.conf = conf + self.log = log + self.inspector_url = inspector_url + + @abc.abstractmethod + def start(self, host): + pass + + @abc.abstractmethod + def stop(self): + pass diff --git a/doctor_tests/monitor/collectd.py b/doctor_tests/monitor/collectd.py new file mode 100644 index 00000000..a22d7edc --- /dev/null +++ b/doctor_tests/monitor/collectd.py @@ -0,0 +1,137 @@ +############################################################################## +# Copyright (c) 2017 NEC Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import socket +import getpass +import sys + +from doctor_tests.monitor.base import BaseMonitor + + +class CollectdMonitor(BaseMonitor): + def __init__(self, conf, inspector_url, log): + super(CollectdMonitor, self).__init__(conf, inspector_url, log) + self.top_dir = os.path.dirname(sys.path[0]) + tmp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + tmp_sock.connect(("8.8.8.8", 80)) + + ## control_ip is the IP of primary interface of control node i.e. + ## eth0, eno1. It is used by collectd monitor to communicate with + ## sample inspector. + ## TODO (umar) see if mgmt IP of control is a better option. Also + ## primary interface may not be the right option + self.control_ip = tmp_sock.getsockname()[0] + self.compute_user = getpass.getuser() + self.interface_name = os.environ.get('INTERFACE_NAME') or '' + self.inspector_type = os.environ.get('INSPECTOR_TYPE', 'sample') + self.auth_url = os.environ.get('OS_AUTH_URL') + self.username = os.environ.get('OS_USERNAME') + self.password = os.environ.get('OS_PASSWORD') + self.project_name = os.environ.get('OS_PROJECT_NAME') + self.user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default' + self.user_domain_id = os.environ.get('OS_USER_DOMAIN_ID') + self.project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default' + self.project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID') + self.ssh_opts_cpu = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + + def start(self, host): + self.log.info("Collectd monitor start.........") + self.compute_host = host.name + self.compute_ip = host.ip + f = open("%s/tests/collectd.conf" % self.top_dir, 'w') + collectd_conf_file = """ +Hostname %s +FQDNLookup false +Interval 1 +MaxReadInterval 2 + +<LoadPlugin python> +Globals true +</LoadPlugin> +LoadPlugin ovs_events +LoadPlugin logfile + +<Plugin logfile> + File \"/var/log/collectd.log\" + Timestamp true + LogLevel \"info\" +</Plugin> + +<Plugin python> + ModulePath \"/home/%s\" + LogTraces true + Interactive false + Import \"collectd_plugin\" + <Module \"collectd_plugin\"> + control_ip \"%s\" + compute_ip \"%s\" + compute_host \"%s\" + compute_user \"%s\" + inspector_type \"%s\" + os_auth_url \"%s\" + os_username \"%s\" + os_password \"%s\" + os_project_name \"%s\" + os_user_domain_name \"%s\" + os_user_domain_id \"%s\" + os_project_domain_name \"%s\" + os_project_domain_id \"%s\" + </Module> +</Plugin> + +<Plugin ovs_events> + Port 6640 + Socket \"/var/run/openvswitch/db.sock\" + Interfaces \"@INTERFACE_NAME@\" + SendNotification true + DispatchValues false +</Plugin> + """ % (self.compute_host, self.compute_user, self.control_ip, self.compute_ip, self.compute_host, self.compute_user, + self.inspector_type, self.auth_url, self.username, self.password, self.project_name, self.user_domain_name, + self.user_domain_id, self.project_domain_name, self.project_domain_id) + f.write(collectd_conf_file) + f.close() + + os.system(" scp %s %s/tests/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip)) + self.log.info("after first scp") + ## @TODO (umar) Always assuming that the interface is assigned an IP if + ## interface name is not provided. See if there is a better approach + os.system(""" ssh %s %s@%s \"if [ -n \"%s\" ]; then + dev=%s + else + dev=\$(sudo ip a | awk '/ %s\//{print \$NF}') + fi + sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf + collectd_conf=/opt/collectd/etc/collectd.conf + if [ -e \$collectd_conf ]; then + sudo cp \$collectd_conf \${collectd_conf}-doctor-saved + else + sudo touch \${collectd_conf}-doctor-created + fi + sudo mv collectd.conf /opt/collectd/etc/collectd.conf\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip, self.interface_name, self.interface_name, self.compute_ip)) + self.log.info("after first ssh") + os.system(" scp %s %s/tests/lib/monitors/collectd/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip)) + self.log.info("after sec scp") + os.system(" ssh %s %s@%s \"sudo pkill collectd; sudo /opt/collectd/sbin/collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip)) + self.log.info("after sec ssh") + + def stop(self): + os.system(" ssh %s %s@%s \"sudo pkill collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip)) + + def cleanup(self): + os.system(""" ssh %s %s@%s \" + collectd_conf=/opt/collectd/etc/collectd.conf + if [ -e \"\${collectd_conf}-doctor-created\" ]; then + sudo rm \"\${collectd_conf}-doctor-created\" + sudo rm \$collectd_conf + elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then + sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf + sudo rm \"\${collectd_conf}-doctor-saved\" + fi\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip)) + os.remove("%s/tests/collectd.conf" % self.top_dir) diff --git a/doctor_tests/monitor/sample.py b/doctor_tests/monitor/sample.py new file mode 100644 index 00000000..7a463048 --- /dev/null +++ b/doctor_tests/monitor/sample.py @@ -0,0 +1,106 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from datetime import datetime +import json +import requests +import socket +from threading import Thread +import time + +from doctor_tests.identity_auth import get_session +from doctor_tests.monitor.base import BaseMonitor + + +class SampleMonitor(BaseMonitor): + event_type = "compute.host.down" + + def __init__(self, conf, inspector_url, log): + super(SampleMonitor, self).__init__(conf, inspector_url, log) + self.session = get_session() + self.pinger = None + + def start(self, host): + self.log.info('sample monitor start......') + self.pinger = Pinger(host.name, host.ip, self, self.log) + self.pinger.start() + + def stop(self): + self.log.info('sample monitor stop......') + if self.pinger is not None: + self.pinger.stop() + self.pinger.join() + + def report_error(self, hostname): + self.log.info('sample monitor report error......') + data = [ + { + 'id': 'monitor_sample_id1', + 'time': datetime.now().isoformat(), + 'type': self.event_type, + 'details': { + 'hostname': hostname, + 'status': 'down', + 'monitor': 'monitor_sample', + 'monitor_event_id': 'monitor_sample_event1' + }, + }, + ] + + auth_token = self.session.get_token() if \ + self.conf.inspector.type != 'sample' else None + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'X-Auth-Token': auth_token, + } + + url = '%s%s' % (self.inspector_url, 'events') \ + if self.inspector_url.endswith('/') else \ + '%s%s' % (self.inspector_url, '/events') + requests.put(url, data=json.dumps(data), headers=headers) + + +class Pinger(Thread): + interval = 0.1 # second + timeout = 0.1 # second + ICMP_ECHO_MESSAGE = bytes([0x08, 0x00, 0xf7, 0xff, 0x00, 0x00, 0x00, 0x00]) + + def __init__(self, host_name, host_ip, monitor, log): + Thread.__init__(self) + self.monitor = monitor + self.hostname = host_name + self.ip_addr = host_ip or socket.gethostbyname(self.hostname) + self.log = log + self._stopped = False + + def run(self): + self.log.info("Starting Pinger host_name(%s), host_ip(%s)" + % (self.hostname, self.ip_addr)) + + sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_ICMP) + sock.settimeout(self.timeout) + while True: + if self._stopped: + return + try: + sock.sendto(self.ICMP_ECHO_MESSAGE, (self.ip_addr, 0)) + sock.recv(4096) + except socket.timeout: + self.log.info("doctor monitor detected at %s" % time.time()) + self.monitor.report_error(self.hostname) + self.log.info("ping timeout, quit monitoring...") + self._stopped = True + return + time.sleep(self.interval) + + def stop(self): + self.log.info("Stopping Pinger host_name(%s), host_ip(%s)" + % (self.hostname, self.ip_addr)) + self._stopped = True diff --git a/doctor_tests/network.py b/doctor_tests/network.py new file mode 100644 index 00000000..ee153e66 --- /dev/null +++ b/doctor_tests/network.py @@ -0,0 +1,68 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from oslo_config import cfg + +from doctor_tests.identity_auth import get_identity_auth +from doctor_tests.identity_auth import get_session +from doctor_tests.os_clients import neutron_client + + +OPTS = [ + cfg.StrOpt('net_name', + default='doctor_net', + help='the name of test net', + required=True), + cfg.StrOpt('net_cidr', + default='192.168.168.0/24', + help='the cidr of test subnet', + required=True), +] + + +class Network(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.auth = get_identity_auth(username=self.conf.doctor_user, + password=self.conf.doctor_passwd, + project=self.conf.doctor_project) + self.neutron = neutron_client(get_session(auth=self.auth)) + self.net = None + self.subnet = None + + def create(self): + self.log.info('network create start.......') + net_name = self.conf.net_name + networks = self.neutron.list_networks(name=net_name)['networks'] + self.net = networks[0] if networks \ + else self.neutron.create_network( + {'network': {'name': net_name}})['network'] + self.log.info('network create end.......') + + self.log.info('subnet create start.......') + subnets = self.neutron.list_subnets(network_id=self.net['id'])['subnets'] + subnet_param = {'name': net_name, 'network_id': self.net['id'], + 'cidr': self.conf.net_cidr, 'ip_version': 4, + 'enable_dhcp': False} + self.subnet = subnets[0] if subnets \ + else self.neutron.create_subnet( + {'subnet': subnet_param})['subnet'] + self.log.info('subnet create end.......') + + def delete(self): + self.log.info('subnet delete start.......') + if self.subnet: + self.neutron.delete_subnet(self.subnet['id']) + self.log.info('subnet delete end.......') + + self.log.info('network delete start.......') + if self.net: + self.neutron.delete_network(self.net['id']) + self.log.info('network delete end.......') diff --git a/doctor_tests/os_clients.py b/doctor_tests/os_clients.py new file mode 100644 index 00000000..44fa3aad --- /dev/null +++ b/doctor_tests/os_clients.py @@ -0,0 +1,50 @@ +##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+import aodhclient.client as aodhclient
+from congressclient.v1 import client as congressclient
+import glanceclient.client as glanceclient
+from keystoneclient.v2_0 import client as ks_client
+from neutronclient.v2_0 import client as neutronclient
+import novaclient.client as novaclient
+
+
+OPTS = [
+ cfg.StrOpt('glance_version', default='2', help='glance version'),
+ cfg.StrOpt('nova_version', default='2.34', help='Nova version'),
+ cfg.StrOpt('aodh_version', default='2', help='aodh version'),
+]
+
+
+def glance_client(version, session):
+ return glanceclient.Client(version=version,
+ session=session)
+
+
+def keystone_client(session):
+ return ks_client.Client(session=session)
+
+
+def nova_client(version, session):
+ return novaclient.Client(version=version,
+ session=session)
+
+
+def neutron_client(session):
+ return neutronclient.Client(session=session)
+
+
+def aodh_client(version, session):
+ return aodhclient.Client(version, session=session)
+
+
+def congress_client(session):
+ return congressclient.Client(session=session,
+ service_type='policy')
diff --git a/doctor_tests/profiler_poc.py b/doctor_tests/profiler_poc.py new file mode 100644 index 00000000..ea36eaeb --- /dev/null +++ b/doctor_tests/profiler_poc.py @@ -0,0 +1,100 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +""" +PoC of performance profiler for OPNFV doctor project + +Usage: + +Export environment variables to set timestamp at each checkpoint in millisecond. +Valid check points are: DOCTOR_PROFILER_T{00-09} + +See also: https://goo.gl/98Osig +""" + +import json +import os + +from oslo_config import cfg + + +OPTS = [ + cfg.StrOpt('profiler_type', + default=os.environ.get('PROFILER_TYPE', 'poc'), + help='the type of installer'), +] + + +OUTPUT = 'doctor_profiling_output' +PREFIX = 'DOCTOR_PROFILER' +TOTAL_CHECK_POINTS = 10 +MODULE_CHECK_POINTS = ['T00', 'T01', 'T04', 'T05', 'T06', 'T09'] +TAG_FORMAT = "{:<5}" +# Inspired by https://github.com/reorx/httpstat +TEMPLATE = """ +Total time cost: {total}(ms) +==============================================================================> + |Monitor|Inspector |Controller|Notifier|Evaluator | + |{M00} |{M01} |{M02} |{M03} |{M04} | + | | | | | | | | | | +link down:{T00}| | | | | | | | | + raw failure:{T01}| | | | | | | | + found affected:{T02}| | | | | | | + set VM error:{T03}| | | | | | + marked host down:{T04}| | | | | + notified VM error:{T05} | | | | + transformed event:{T06}| | | + evaluated event:{T07}| | + fired alarm:{T08}| + received alarm:{T09} +""" + + +def main(log=None): + check_points = ["T{:02d}".format(i) for i in range(TOTAL_CHECK_POINTS)] + module_map = {"M{:02d}".format(i): + (MODULE_CHECK_POINTS[i], MODULE_CHECK_POINTS[i + 1]) + for i in range(len(MODULE_CHECK_POINTS) - 1)} + + # check point tags + elapsed_ms = {cp: os.getenv("{}_{}".format(PREFIX, cp)) + for cp in check_points} + + def format_tag(tag): + return TAG_FORMAT.format(tag or '?') + + tags = {cp: format_tag(ms) for cp, ms in elapsed_ms.items()} + + def time_cost(cp): + if elapsed_ms[cp[0]] and elapsed_ms[cp[1]]: + return int(elapsed_ms[cp[1]]) - int(elapsed_ms[cp[0]]) + else: + return None + + # module time cost tags + modules_cost_ms = {module: time_cost(cp) + for module, cp in module_map.items()} + + tags.update({module: format_tag(cost) + for module, cost in modules_cost_ms.items()}) + + tags.update({'total': time_cost((check_points[0], check_points[-1]))}) + + profile = TEMPLATE.format(**tags) + + logfile = open('{}.json'.format(OUTPUT), 'w') + logfile.write(json.dumps(tags)) + + print(profile) + if log: + log.info('%s' % profile) + + +if __name__ == '__main__': + main() diff --git a/doctor_tests/scenario/__init__.py b/doctor_tests/scenario/__init__.py new file mode 100644 index 00000000..48893ae6 --- /dev/null +++ b/doctor_tests/scenario/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/doctor_tests/scenario/common.py b/doctor_tests/scenario/common.py new file mode 100644 index 00000000..a5cbe483 --- /dev/null +++ b/doctor_tests/scenario/common.py @@ -0,0 +1,29 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import sys + +from doctor_tests.common.utils import match_rep_in_file + + +def calculate_notification_time(): + log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log') + + reg = '(?<=doctor monitor detected at )\d+.\d+' + result = match_rep_in_file(reg, log_file) + if not result: + raise Exception('Can not match detected time') + detected = result.group(0) + + reg = '(?<=doctor consumer notified at )\d+.\d+' + result = match_rep_in_file(reg, log_file) + if not result: + raise Exception('Can not match notified time') + notified = result.group(0) + + return float(notified) - float(detected) diff --git a/doctor_tests/scenario/network_failure.py b/doctor_tests/scenario/network_failure.py new file mode 100644 index 00000000..b94a622d --- /dev/null +++ b/doctor_tests/scenario/network_failure.py @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from doctor_tests.identity_auth import get_session +from doctor_tests.os_clients import nova_client +from doctor_tests.common.utils import SSHClient + +LINK_DOWN_SCRIPT = """ +#!/bin/bash -x +dev=$(sudo ip a | awk '/ {compute_ip}\//{{print $NF}}') +sleep 1 +sudo ip link set $dev down +echo "doctor set link down at" $(date "+%s.%N") +sleep 10 +sudo ip link set $dev up +sleep 1 +""" + + +class NetworkFault(object): + + def __init__(self, conf, installer, log): + self.conf = conf + self.log = log + self.installer = installer + self.nova = nova_client(self.conf.nova_version, get_session()) + self.host = None + self.GetLog = False + + def start(self, host): + self.log.info('fault inject start......') + self._set_link_down(host.ip) + self.host = host + self.log.info('fault inject end......') + + def cleanup(self): + self.log.info('fault inject cleanup......') + self.get_disable_network_log() + + def get_disable_network_log(self): + if self.GetLog: + self.log.info('Already get the disable_netork.log from down_host......') + return + if self.host is not None: + client = SSHClient(self.host.ip, + self.installer.node_user_name, + key_filename=self.installer.get_ssh_key_from_installer(), + look_for_keys=True, + log=self.log) + client.scp('disable_network.log', './disable_network.log', method='get') + self.log.info('Get the disable_netork.log from down_host(host_name:%s, host_ip:%s)' + % (self.host.name, self.host.ip)) + self.GetLog = True + + def _set_link_down(self, compute_ip): + file_name = './disable_network.sh' + with open(file_name, 'w') as file: + file.write(LINK_DOWN_SCRIPT.format(compute_ip=compute_ip)) + client = SSHClient(compute_ip, + self.installer.node_user_name, + key_filename=self.installer.get_ssh_key_from_installer(), + look_for_keys=True, + log=self.log) + client.scp('./disable_network.sh', 'disable_network.sh') + command = 'bash disable_network.sh > disable_network.log 2>&1 &' + client.ssh(command) diff --git a/doctor_tests/user.py b/doctor_tests/user.py new file mode 100644 index 00000000..33f995e7 --- /dev/null +++ b/doctor_tests/user.py @@ -0,0 +1,163 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os + +from oslo_config import cfg + +from doctor_tests.identity_auth import get_session +from doctor_tests.os_clients import keystone_client +from doctor_tests.os_clients import nova_client + + +OPTS = [ + cfg.StrOpt('doctor_user', + default='doctor', + help='the name of test user', + required=True), + cfg.StrOpt('doctor_passwd', + default='doctor', + help='the password of test user', + required=True), + cfg.StrOpt('doctor_project', + default='doctor', + help='the name of test project', + required=True), + cfg.StrOpt('doctor_role', + default='_member_', + help='the role of test user', + required=True), + cfg.IntOpt('quota_instances', + default=os.environ.get('VM_COUNT', 1), + help='the quota of instances in test user', + required=True), + cfg.IntOpt('quota_cores', + default=os.environ.get('VM_COUNT', 1), + help='the quota of cores in test user', + required=True), +] + + +class User(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.keystone = \ + keystone_client(get_session()) + self.nova = \ + nova_client(conf.nova_version, get_session()) + self.users = {} + self.projects = {} + self.roles = {} + self.roles_for_user = {} + self.roles_for_admin = {} + + def create(self): + """create test user, project and etc""" + self.log.info('user create start......') + + self._create_project() + self._create_user() + self._create_role() + self._add_user_role_in_project(is_admin=False) + self._add_user_role_in_project(is_admin=True) + + self.log.info('user create end......') + + def _create_project(self): + """create test project""" + self.projects = {project.name: project + for project in self.keystone.tenants.list()} + if self.conf.doctor_project not in self.projects: + test_project = \ + self.keystone.tenants.create(self.conf.doctor_project) + self.projects[test_project.name] = test_project + + def _create_user(self): + """create test user""" + project = self.projects.get(self.conf.doctor_project) + self.users = {user.name: user for user in self.keystone.users.list()} + if self.conf.doctor_user not in self.users: + test_user = self.keystone.users.create( + self.conf.doctor_user, + password=self.conf.doctor_passwd, + tenant_id=project.id) + self.users[test_user.name] = test_user + + def _create_role(self): + """create test role""" + self.roles = {role.name: role for role in self.keystone.roles.list()} + if self.conf.doctor_role not in self.roles: + test_role = self.keystone.roles.create(self.conf.doctor_role) + self.roles[test_role.name] = test_role + + def _add_user_role_in_project(self, is_admin=False): + """add test user with test role in test project""" + project = self.projects.get(self.conf.doctor_project) + + user_name = 'admin' if is_admin else self.conf.doctor_user + user = self.users.get(user_name) + + role_name = 'admin' if is_admin else self.conf.doctor_role + role = self.roles.get(role_name) + + roles_for_user = self.roles_for_admin \ + if is_admin else self.roles_for_user + + roles_for_user = \ + {role.name: role for role in + self.keystone.roles.roles_for_user(user, tenant=project)} + if role_name not in roles_for_user: + self.keystone.roles.add_user_role(user, role, tenant=project) + roles_for_user[role_name] = role + + def delete(self): + """delete the test user, project and role""" + self.log.info('user delete start......') + + project = self.projects.get(self.conf.doctor_project) + user = self.users.get(self.conf.doctor_user) + role = self.roles.get(self.conf.doctor_role) + + if project: + if 'admin' in self.roles_for_admin: + self.keystone.roles.remove_user_role( + self.users['admin'], + self.roles['admin'], + tenant=project) + + if user: + if role and self.conf.doctor_role in self.roles_for_user: + self.keystone.roles.remove_user_role( + user, role, tenant=project) + self.keystone.roles.delete(role) + self.keystone.users.delete(user) + + self.keystone.tenants.delete(project) + self.log.info('user delete end......') + + def update_quota(self): + self.log.info('user quota update start......') + project = self.projects.get(self.conf.doctor_project) + user = self.users.get(self.conf.doctor_user) + + if project and user: + self.quota = self.nova.quotas.get(project.id, + user_id=user.id) + if self.conf.quota_instances > self.quota.instances: + self.nova.quotas.update(project.id, + instances=self.conf.quota_instances, + user_id=user.id) + if self.conf.quota_cores > self.quota.cores: + self.nova.quotas.update(project.id, + cores=self.conf.quota_cores, + user_id=user.id) + self.log.info('user quota update end......') + else: + raise Exception('No project or role for update quota') |