diff options
46 files changed, 2359 insertions, 95 deletions
@@ -1,4 +1,5 @@ *~ +*.pyc .*.sw? **.log /docs_build/ @@ -6,5 +7,8 @@ /releng/ /tests/*.img +#IntelJ Idea +.idea/ + #Build results .tox @@ -49,4 +49,6 @@ - url: https://review.openstack.org/424340 system: Gerrit - # WIP +- + url: https://bugs.launchpad.net/python-openstackclient/+bug/1684989 + system: Launchpad-bug diff --git a/devstack/README.rst b/devstack/README.rst index cd836f13..c55d5626 100644 --- a/devstack/README.rst +++ b/devstack/README.rst @@ -9,7 +9,7 @@ Enabling OPNFV Doctor using DevStack This directory contains the files necessary to run OpenStack with enabled OPNFV Doctor in DevStack. -To configure DevStack to enable OPNFV Doctor edit +1. To configure DevStack to enable OPNFV Doctor edit ``${DEVSTACK_DIR}/local.conf`` file and add:: enable_plugin aodh http://git.openstack.org/openstack/aodh @@ -22,6 +22,14 @@ to the ``[[local|localrc]]`` section. .. note:: The order of enabling plugins matters. -Run DevStack as normal:: +2. To enable Python 3 in DevStack, please add:: + + USE_PYTHON3=True + +3. To enable Congress as Doctor Inspector, please also add:: + + enable_plugin congress https://git.openstack.org/openstack/congress + +4. Run DevStack as normal:: $ ./stack.sh diff --git a/docs/development/manuals/monitors.rst b/docs/development/manuals/monitors.rst new file mode 100644 index 00000000..0d22b1de --- /dev/null +++ b/docs/development/manuals/monitors.rst @@ -0,0 +1,36 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + +Monitor Types and Limitations +============================= + +Currently there are two monitor types supported: sample and collectd + +Sample Monitor +-------------- + +Sample monitor type pings the compute host from the control host and calculates the +notification time after the ping timeout. +Also if inspector type is sample, the compute node needs to communicate with the control +node on port 12345. This port needs to be opened for incomming traffic on control node. + +Collectd Monitor +---------------- + +Collectd monitor type uses collectd daemon running ovs_events plugin. Collectd runs on +compute to send instant notification to the control node. The notification time is +calculated by using the difference of time at which compute node sends notification to +control node and the time at which consumer is notified. The time on control and compute +node has to be synchronized for this reason. For further details on setting up collectd +on the compute node, use the following link: +http://docs.opnfv.org/en/stable-danube/submodules/barometer/docs/release/userguide/feature.userguide.html#id18 + +Collectd monitors an interface managed by OVS. If the interface is not be assigned +an IP, the user has to provide the name of interface to be monitored. The command to +launch the doctor test in that case is: +MONITOR_TYPE=collectd INSPECTOR_TYPE=sample INTERFACE_NAME=example_iface ./run.sh + +If the interface name or IP is not provided, the collectd monitor type will monitor the +default management interface. This may result in the failure of doctor run.sh test case. +The test case sets the monitored interface down and if the inspector (sample or congress) +is running on the same subnet, collectd monitor will not be able to communicate with it. diff --git a/etc/doctor.sample.conf b/etc/doctor.sample.conf new file mode 100644 index 00000000..52d78d6b --- /dev/null +++ b/etc/doctor.sample.conf @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +[DEFAULT] +#image_name = cirros +#image_format = qcow2 +#image_filename = cirros.img +#image_download_url = https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img + +#glance_version = 2 +#nova_version = 2.34 +#aodh_version = 2 + +#doctor_user = doctor +#doctor_passwd = doctor +#doctor_project = doctor +#doctor_role = _member_ +#quota_instances = 1 +#quota_cores = 1 + +#net_name = doctor_net +#net_cidr = 192.168.168.0/24 +#flavor = m1.tiny +#instance_count = 1 +#instance_basename = doctor_vm + +#alarm_basename = doctor_alarm + +[installer] +#type = local +#ip = 127.0.0.1 +#username = root + +[monitor] +#type = sample + +[inspector] +#type = sample +#ip = 127.0.0.1 +#port = 12345 + +[consumer] +#type = sample +#ip = 127.0.0.1 +#port = 12346 diff --git a/test-requirements.txt b/test-requirements.txt index 2928e0f7..070caa44 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,6 +5,7 @@ requests>=2.8.0 oslo.config==3.22.0 # Apache-2.0 python-openstackclient==2.3.0 python-ceilometerclient==2.6.2 +aodhclient==0.7.0 python-keystoneclient==3.5.0 python-neutronclient==6.0.0 python-novaclient==6.0.0 diff --git a/tests/alarm.py b/tests/alarm.py new file mode 100644 index 00000000..0582085e --- /dev/null +++ b/tests/alarm.py @@ -0,0 +1,95 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from oslo_config import cfg + +from identity_auth import get_identity_auth +from identity_auth import get_session +from os_clients import aodh_client +from os_clients import nova_client + +OPTS = [ + cfg.StrOpt('alarm_basename', + default='doctor_alarm', + help='the base name of alarm', + required=True), +] + + +class Alarm(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.auth = get_identity_auth(username=self.conf.doctor_user, + password=self.conf.doctor_passwd, + project=self.conf.doctor_project) + self.aodh = \ + aodh_client(conf.aodh_version, + get_session(auth=self.auth)) + self.nova = \ + nova_client(conf.nova_version, + get_session(auth=self.auth)) + self._init_alarm_name() + + def _init_alarm_name(self): + self.alarm_names = [] + for i in range(0, self.conf.instance_count): + alarm_name = '%s%d' % (self.conf.alarm_basename, i) + self.alarm_names.append(alarm_name) + + def create(self): + self.log.info('alarm create start......') + + alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()} + servers = \ + {getattr(server, 'name'): server + for server in self.nova.servers.list()} + + for i in range(0, self.conf.instance_count): + alarm_name = self.alarm_names[i] + if alarm_name in alarms: + continue; + vm_name = '%s%d' % (self.conf.instance_basename, i) + vm_id = getattr(servers[vm_name], 'id') + alarm_request = dict( + name=alarm_name, + description=u'VM failure', + enabled=True, + alarm_actions=[u'http://%s:%d/failure' + % (self.conf.consumer.ip, + self.conf.consumer.port)], + repeat_actions=False, + severity=u'moderate', + type=u'event', + event_rule=dict( + event_type=u'compute.instance.update', + query=[ + dict(field=u'traits.instance_id', + type='', + op=u'eq', + value=vm_id), + dict(field=u'traits.state', + type='', + op=u'eq', + value=u'error')])) + self.aodh.alarm.create(alarm_request) + + self.log.info('alarm create end......') + + def delete(self): + self.log.info('alarm delete start.......') + + alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()} + for alarm_name in self.alarm_names: + if alarm_name in alarms: + self.aodh.alarm.delete(alarms[alarm_name]['alarm_id']) + + del self.alarm_names[:] + + self.log.info('alarm delete end.......') diff --git a/tests/config.py b/tests/config.py index 2288d36e..f33ab5d2 100644 --- a/tests/config.py +++ b/tests/config.py @@ -6,20 +6,39 @@ # which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+import itertools
+
from oslo_config import cfg
+import alarm
+import consumer
import image
+import instance
+import installer
+import network
+import inspector
+import monitor
import os_clients
+import user
def list_opts():
return [
- ('os_clients', os_clients.OPTS),
- ('image', image.IMAGE_OPTS),
+ ('installer', installer.OPTS),
+ ('monitor', monitor.OPTS),
+ ('inspector', inspector.OPTS),
+ ('consumer', consumer.OPTS),
+ ('DEFAULT', itertools.chain(
+ os_clients.OPTS,
+ image.OPTS,
+ user.OPTS,
+ network.OPTS,
+ instance.OPTS,
+ alarm.OPTS))
]
-def prepare_conf(conf=None):
+def prepare_conf(args=None, conf=None, config_files=None):
if conf is None:
conf = cfg.ConfigOpts()
@@ -27,4 +46,7 @@ def prepare_conf(conf=None): conf.register_opts(list(options),
group=None if group == 'DEFAULT' else group)
+ conf(args, project='doctor', validate_default_values=True,
+ default_config_files=config_files)
+
return conf
diff --git a/tests/consumer/__init__.py b/tests/consumer/__init__.py new file mode 100644 index 00000000..ccec8644 --- /dev/null +++ b/tests/consumer/__init__.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from oslo_config import cfg +from oslo_utils import importutils + + +OPTS = [ + cfg.StrOpt('type', + default='sample', + choices=['sample'], + help='the component of doctor consumer', + required=True), + cfg.StrOpt('ip', + default='127.0.0.1', + help='the ip of consumer', + required=True), + cfg.IntOpt('port', + default='12346', + help='the port of doctor consumer', + required=True), +] + + +_consumer_name_class_mapping = { + 'sample': 'consumer.sample.SampleConsumer' +} + + +def get_consumer(conf, log): + consumer_class = _consumer_name_class_mapping.get(conf.consumer.type) + return importutils.import_object(consumer_class, conf, log)
\ No newline at end of file diff --git a/tests/consumer/base.py b/tests/consumer/base.py new file mode 100644 index 00000000..35170748 --- /dev/null +++ b/tests/consumer/base.py @@ -0,0 +1,26 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseConsumer(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + + @abc.abstractmethod + def start(self): + pass + + @abc.abstractmethod + def stop(self): + pass
\ No newline at end of file diff --git a/tests/consumer/sample.py b/tests/consumer/sample.py new file mode 100644 index 00000000..a698623a --- /dev/null +++ b/tests/consumer/sample.py @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from flask import Flask +from flask import request +import json +import time +from threading import Thread +import requests + +from consumer.base import BaseConsumer + + +class SampleConsumer(BaseConsumer): + + def __init__(self, conf, log): + super(SampleConsumer, self).__init__(conf, log) + self.app = None + + def start(self): + self.log.info('sample consumer start......') + self.app = ConsumerApp(self.conf.consumer.port, self, self.log) + self.app.start() + + def stop(self): + self.log.info('sample consumer stop......') + if not self.app: + return + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + url = 'http://%s:%d/shutdown'\ + % (self.conf.consumer.ip, + self.conf.consumer.port) + requests.post(url, data='', headers=headers) + + +class ConsumerApp(Thread): + + def __init__(self, port, consumer, log): + Thread.__init__(self) + self.port = port + self.consumer = consumer + self.log = log + + def run(self): + app = Flask('consumer') + + @app.route('/failure', methods=['POST']) + def event_posted(): + self.log.info('doctor consumer notified at %s' % time.time()) + self.log.info('received data = %s' % request.data) + data = json.loads(request.data) + return "OK" + + @app.route('/shutdown', methods=['POST']) + def shutdown(): + self.log.info('shutdown consumer app server at %s' % time.time()) + func = request.environ.get('werkzeug.server.shutdown') + if func is None: + raise RuntimeError('Not running with the Werkzeug Server') + func() + return 'consumer app shutting down...' + + app.run(host="0.0.0.0", port=self.port)
\ No newline at end of file diff --git a/tests/identity_auth.py b/tests/identity_auth.py index ffecc68a..c94893f4 100644 --- a/tests/identity_auth.py +++ b/tests/identity_auth.py @@ -13,13 +13,14 @@ from keystoneauth1 import loading from keystoneauth1 import session -def get_identity_auth(): +def get_identity_auth(username=None, password=None, project=None): auth_url = os.environ['OS_AUTH_URL'] - username = os.environ['OS_USERNAME'] - password = os.environ['OS_PASSWORD'] + username = username or os.environ['OS_USERNAME'] + password = password or os.environ['OS_PASSWORD'] user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default' user_domain_id = os.environ.get('OS_USER_DOMAIN_ID') or 'default' - project_name = os.environ.get('OS_PROJECT_NAME') or os.environ.get('OS_TENANT_NAME') + project_name = project or os.environ.get('OS_PROJECT_NAME') \ + or os.environ.get('OS_TENANT_NAME') project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default' project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID') or 'default' diff --git a/tests/image.py b/tests/image.py index 0b4a3d72..453322b8 100644 --- a/tests/image.py +++ b/tests/image.py @@ -7,71 +7,68 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import os -import urllib2 +import urllib.request from oslo_config import cfg from identity_auth import get_session from os_clients import glance_client -import logger as doctor_log -IMAGE_OPTS = [ - cfg.StrOpt('name', +OPTS = [ + cfg.StrOpt('image_name', default=os.environ.get('IMAGE_NAME', 'cirros'), help='the name of test image', required=True), - cfg.StrOpt('format', + cfg.StrOpt('image_format', default='qcow2', help='the format of test image', required=True), - cfg.StrOpt('file_name', + cfg.StrOpt('image_filename', default='cirros.img', help='the name of image file', required=True), - cfg.StrOpt('url', + cfg.StrOpt('image_download_url', default='https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img', help='the url where to get the image', required=True), ] -LOG = doctor_log.Logger('doctor').getLogger() - class Image(object): - def __init__(self, conf): + def __init__(self, conf, log): self.conf = conf + self.log = log self.glance = \ - glance_client(conf.os_clients.glance_version, - get_session()) + glance_client(conf.glance_version, get_session()) self.use_existing_image = False self.image = None def create(self): - LOG.info('image create start......') + self.log.info('image create start......') images = {image.name: image for image in self.glance.images.list()} - if self.conf.image.name not in images: - if not os.path.exists(self.conf.image.file_name): - resp = urllib2.urlopen(self.conf.image.url) - with open(self.conf.image.file_name, "wb") as file: + if self.conf.image_name not in images: + if not os.path.exists(self.conf.image_filename): + resp = urllib.request.urlopen(self.conf.image_download_url) + with open(self.conf.image_filename, "wb") as file: file.write(resp.read()) - self.image = self.glance.images.create(name=self.conf.image.name, - disk_format=self.conf.image.format, + self.image = self.glance.images.create(name=self.conf.image_name, + disk_format=self.conf.image_format, container_format="bare", visibility="public") self.glance.images.upload(self.image['id'], - open(self.conf.image.file_name, 'rb')) + open(self.conf.image_filename, 'rb')) else: self.use_existing_image = True - self.image = images[self.conf.image.name] + self.image = images[self.conf.image_name] - LOG.info('image create end......') + self.log.info('image create end......') def delete(self): - LOG.info('image delete start.......') + self.log.info('image delete start.......') if not self.use_existing_image and self.image: self.glance.images.delete(self.image['id']) - LOG.info('image delete end.......') + self.log.info('image delete end.......') diff --git a/tests/inspector.py b/tests/inspector.py index d11da299..82ffc338 100644 --- a/tests/inspector.py +++ b/tests/inspector.py @@ -54,8 +54,7 @@ class DoctorInspectorSample(object): # Pool of novaclients for redundant usage for i in range(self.NUMBER_OF_CLIENTS): self.novaclients.append( - novaclient.Client(self.NOVA_API_VERSION, session=sess, - connection_pool=True)) + novaclient.Client(self.NOVA_API_VERSION, session=sess)) # Normally we use this client for non redundant API calls self.nova=self.novaclients[0] self.nova.servers.list(detailed=False) @@ -117,8 +116,7 @@ def get_args(): def main(): args = get_args() - app.run(port=args.port) - + app.run(host='0.0.0.0', port=args.port) if __name__ == '__main__': main() diff --git a/tests/inspector/__init__.py b/tests/inspector/__init__.py new file mode 100644 index 00000000..afba4800 --- /dev/null +++ b/tests/inspector/__init__.py @@ -0,0 +1,40 @@ +############################################################################# +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os + +from oslo_config import cfg +from oslo_utils import importutils + + +OPTS = [ + cfg.StrOpt('type', + default=os.environ.get('INSPECTOR_TYPE', 'sample'), + choices=['sample', 'congress', 'vitrage'], + help='the component of doctor inspector', + required=True), + cfg.StrOpt('ip', + default='127.0.0.1', + help='the host ip of inspector', + required=False), + cfg.StrOpt('port', + default='12345', + help='the port of default for inspector', + required=False), +] + + +_inspector_name_class_mapping = { + 'sample': 'inspector.sample.SampleInspector', + 'congress': 'inspector.congress.CongressInspector', +} + + +def get_inspector(conf, log): + inspector_class = _inspector_name_class_mapping[conf.inspector.type] + return importutils.import_object(inspector_class, conf, log) diff --git a/tests/inspector/base.py b/tests/inspector/base.py new file mode 100644 index 00000000..854f0695 --- /dev/null +++ b/tests/inspector/base.py @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseInspector(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + + @abc.abstractmethod + def get_inspector_url(self): + pass + + @abc.abstractmethod + def start(self): + pass + + @abc.abstractmethod + def stop(self): + pass
\ No newline at end of file diff --git a/tests/inspector/congress.py b/tests/inspector/congress.py new file mode 100644 index 00000000..ae295852 --- /dev/null +++ b/tests/inspector/congress.py @@ -0,0 +1,94 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from identity_auth import get_identity_auth +from identity_auth import get_session +from os_clients import congress_client + +from inspector.base import BaseInspector + + +class CongressInspector(BaseInspector): + nova_api_min_version = '2.11' + doctor_driver = 'doctor' + doctor_datasource = 'doctor' + policy = 'classification' + rules = { + 'host_down': + 'host_down(host) :- doctor:events(hostname=host, type="compute.host.down", status="down")', + 'active_instance_in_host': + 'active_instance_in_host(vmid, host) :- nova:servers(id=vmid, host_name=host, status="ACTIVE")', + 'host_force_down': + 'execute[nova:services.force_down(host, "nova-compute", "True")] :- host_down(host)', + 'error_vm_states': + 'execute[nova:servers.reset_state(vmid, "error")] :- host_down(host), active_instance_in_host(vmid, host)' + } + + def __init__(self, conf, log): + super(CongressInspector, self).__init__(conf, log) + self.auth = get_identity_auth() + self.congress = congress_client(get_session(auth=self.auth)) + self._init_driver_and_ds() + self.inspector_url = self.get_inspector_url() + + def _init_driver_and_ds(self): + datasources = \ + {ds['name']: ds for ds in self.congress.list_datasources()['results']} + + # check nova_api version + nova_api_version = datasources['nova']['config'].get('api_version') + if nova_api_version and nova_api_version < self.nova_api_min_version: + raise Exception('Congress Nova datasource API version < nova_api_min_version(%s)' + % self.nova_api_min_version) + + # create doctor datasource if it's not exist + if self.doctor_datasource not in datasources: + self.congress.create_datasource( + body={'driver': self.doctor_driver, + 'name': self.doctor_datasource}) + + # check whether doctor driver exist + drivers = \ + {driver['id']: driver for driver in self.congress.list_drivers()['results']} + if self.doctor_driver not in drivers: + raise Exception('Do not support doctor driver in congress') + + self.policy_rules = \ + {rule['name']: rule for rule in + self.congress.list_policy_rules(self.policy)['results']} + + def get_inspector_url(self): + ds = self.congress.list_datasources()['results'] + doctor_ds = next((item for item in ds if item['driver'] == 'doctor'), + None) + congress_endpoint = self.congress.httpclient.get_endpoint(auth=self.auth) + return ('%s/v1/data-sources/%s/tables/events/rows' % + (congress_endpoint, doctor_ds['id'])) + + def start(self): + self.log.info('congress inspector start......') + + for rule_name, rule in self.rules.items(): + self._add_rule(rule_name, rule) + + def stop(self): + self.log.info('congress inspector stop......') + + for rule_name in self.rules.keys(): + self._del_rule(rule_name) + + def _add_rule(self, rule_name, rule): + if rule_name not in self.policy_rules: + self.congress.create_policy_rule(self.policy, + body={'name': rule_name, + 'rule': rule}) + + def _del_rule(self, rule_name): + if rule_name in self.policy_rules: + rule_id = self.policy_rules[rule_name]['id'] + self.congress.delete_policy_rule(self.policy, rule_id) diff --git a/tests/inspector/sample.py b/tests/inspector/sample.py new file mode 100644 index 00000000..dda053ab --- /dev/null +++ b/tests/inspector/sample.py @@ -0,0 +1,151 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import collections +from flask import Flask +from flask import request +import json +import time +from threading import Thread +import requests + +from identity_auth import get_identity_auth +from identity_auth import get_session +from os_clients import nova_client +from inspector.base import BaseInspector + + +class SampleInspector(BaseInspector): + event_type = 'compute.host.down' + + def __init__(self, conf, log): + super(SampleInspector, self).__init__(conf, log) + self.inspector_url = self.get_inspector_url() + self.novaclients = list() + self._init_novaclients() + # Normally we use this client for non redundant API calls + self.nova = self.novaclients[0] + + self.servers = collections.defaultdict(list) + self.hostnames = list() + self.app = None + + def _init_novaclients(self): + self.NUMBER_OF_CLIENTS = self.conf.instance_count + auth = get_identity_auth(project=self.conf.doctor_project) + session = get_session(auth=auth) + for i in range(self.NUMBER_OF_CLIENTS): + self.novaclients.append( + nova_client(self.conf.nova_version, session)) + + def _init_servers_list(self): + self.servers.clear() + opts = {'all_tenants': True} + servers = self.nova.servers.list(search_opts=opts) + for server in servers: + try: + host = server.__dict__.get('OS-EXT-SRV-ATTR:host') + self.servers[host].append(server) + self.log.debug('get hostname=%s from server=%s' % (host, server)) + except Exception as e: + self.log.info('can not get hostname from server=%s' % server) + + def get_inspector_url(self): + return 'http://%s:%s' % (self.conf.inspector.ip, self.conf.inspector.port) + + def start(self): + self.log.info('sample inspector start......') + self._init_servers_list() + self.app = InspectorApp(self.conf.inspector.port, self, self.log) + self.app.start() + + def stop(self): + self.log.info('sample inspector stop......') + if not self.app: + return + for hostname in self.hostnames: + self.nova.services.force_down(hostname, 'nova-compute', False) + + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + url = '%s%s' % (self.inspector_url, 'shutdown') \ + if self.inspector_url.endswith('/') else \ + '%s%s' % (self.inspector_url, '/shutdown') + requests.post(url, data='', headers=headers) + + def handle_events(self, events): + for event in events: + hostname = event['details']['hostname'] + event_type = event['type'] + if event_type == self.event_type: + self.hostnames.append(hostname) + self.disable_compute_host(hostname) + + def disable_compute_host(self, hostname): + threads = [] + if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS: + # TODO(tojuvone): This could be enhanced in future with dynamic + # reuse of self.novaclients when all threads in use + self.log.error('%d servers in %s. Can handle only %d'%( + self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS)) + for nova, server in zip(self.novaclients, self.servers[hostname]): + t = ThreadedResetState(nova, "error", server, self.log) + t.start() + threads.append(t) + for t in threads: + t.join() + self.nova.services.force_down(hostname, 'nova-compute', True) + self.log.info('doctor mark host(%s) down at %s' % (hostname, time.time())) + + +class ThreadedResetState(Thread): + + def __init__(self, nova, state, server, log): + Thread.__init__(self) + self.nova = nova + self.state = state + self.server = server + self.log = log + + def run(self): + self.nova.servers.reset_state(self.server, self.state) + self.log.info('doctor mark vm(%s) error at %s' % (self.server, time.time())) + + +class InspectorApp(Thread): + + def __init__(self, port, inspector, log): + Thread.__init__(self) + self.port = port + self.inspector = inspector + self.log = log + + def run(self): + app = Flask('inspector') + + @app.route('/events', methods=['PUT']) + def event_posted(): + self.log.info('event posted in sample inspector at %s' % time.time()) + self.log.info('sample inspector = %s' % self.inspector) + self.log.info('sample inspector received data = %s' % request.data) + events = json.loads(request.data.decode('utf8')) + self.inspector.handle_events(events) + return "OK" + + @app.route('/shutdown', methods=['POST']) + def shutdown(): + self.log.info('shutdown inspector app server at %s' % time.time()) + func = request.environ.get('werkzeug.server.shutdown') + if func is None: + raise RuntimeError('Not running with the Werkzeug Server') + func() + return 'inspector app shutting down...' + + app.run(host="0.0.0.0", port=self.port) diff --git a/tests/installer/__init__.py b/tests/installer/__init__.py new file mode 100644 index 00000000..bb0e452d --- /dev/null +++ b/tests/installer/__init__.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os + +from oslo_config import cfg +from oslo_utils import importutils + +OPTS = [ + cfg.StrOpt('type', + default=os.environ.get('INSTALLER_TYPE', 'local'), + choices=['local', 'apex'], + help='the type of installer', + required=True), + cfg.StrOpt('ip', + default=os.environ.get('INSTALLER_IP', '127.0.0.1'), + help='the ip of installer'), + cfg.StrOpt('username', + default='root', + help='the user name for login installer server', + required=True), +] + + +_installer_name_class_mapping = { + 'local': 'installer.local.LocalInstaller', + 'apex': 'installer.apex.ApexInstaller' +} + + +def get_installer(conf, log): + installer_class = _installer_name_class_mapping[conf.installer.type] + return importutils.import_object(installer_class, conf, log)
\ No newline at end of file diff --git a/tests/installer/apex.py b/tests/installer/apex.py new file mode 100644 index 00000000..24cd5a75 --- /dev/null +++ b/tests/installer/apex.py @@ -0,0 +1,99 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import getpass +import grp +import os +import pwd +import stat +import sys + +from installer.common.congress import set_doctor_driver_conf +from installer.common.congress import restore_doctor_driver_conf +from installer.base import BaseInstaller +from utils import SSHClient + + +class ApexInstaller(BaseInstaller): + node_user_name = 'heat-admin' + cm_set_script = 'set_ceilometer.py' + cm_restore_script = 'restore_ceilometer.py' + + def __init__(self, conf, log): + super(ApexInstaller, self).__init__(conf, log) + self.client = SSHClient(self.conf.installer.ip, + self.conf.installer.username, + look_for_keys=True) + self.key_file = None + self.controllers = list() + self.controller_clients = list() + + def setup(self): + self.log.info('Setup Apex installer start......') + + self.key_file = self.get_ssh_key_from_installer() + self.get_controller_ips() + self.set_apply_patches() + + def cleanup(self): + self.restore_apply_patches() + + def get_ssh_key_from_installer(self): + self.log.info('Get SSH keys from Apex installer......') + + self.client.scp('/home/stack/.ssh/id_rsa', './instack_key', method='get') + user = getpass.getuser() + uid = pwd.getpwnam(user).pw_uid + gid = grp.getgrnam(user).gr_gid + os.chown('./instack_key', uid, gid) + os.chmod('./instack_key', stat.S_IREAD) + current_dir = sys.path[0] + return '{0}/{1}'.format(current_dir, 'instack_key') + + def get_controller_ips(self): + self.log.info('Get controller ips from Apex installer......') + + command = "source stackrc; " \ + "nova list | grep ' overcloud-controller-[0-9] ' " \ + "| sed -e 's/^.*ctlplane=//' |awk '{print $1}'" + ret, controllers = self.client.ssh(command) + if ret: + raise Exception('Exec command to get controller ips in Apex installer failed' + 'ret=%s, output=%s' % (ret, controllers)) + self.controllers = controllers + + def set_apply_patches(self): + self.log.info('Set apply patches start......') + + for node_ip in self.controllers: + client = SSHClient(node_ip, self.node_user_name, key_filename=self.key_file) + self.controller_clients.append(client) + self._ceilometer_apply_patches(client, self.cm_set_script) + cmd = 'sudo systemctl restart openstack-congress-server.service' + set_doctor_driver_conf(client, cmd) + + def restore_apply_patches(self): + self.log.info('restore apply patches start......') + + for client in self.controller_clients: + self._ceilometer_apply_patches(client, self.cm_restore_script) + cmd = 'sudo systemctl restart openstack-congress-server.service' + restore_doctor_driver_conf(client, cmd) + + def _ceilometer_apply_patches(self, ssh_client, script_name): + installer_dir = os.path.dirname(os.path.realpath(__file__)) + script_abs_path = '{0}/{1}/{2}'.format(installer_dir, 'common', script_name) + + ssh_client.scp(script_abs_path, script_name) + cmd = 'sudo python %s' % script_name + ret, output = ssh_client.ssh(cmd) + if ret: + raise Exception('Do the ceilometer command in controller node failed....' + 'ret=%s, cmd=%s, output=%s' % (ret, cmd, output)) + ssh_client.ssh('sudo systemctl restart openstack-ceilometer-notification.service') + diff --git a/tests/installer/base.py b/tests/installer/base.py new file mode 100644 index 00000000..f3837f15 --- /dev/null +++ b/tests/installer/base.py @@ -0,0 +1,32 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import abc +import six + +@six.add_metaclass(abc.ABCMeta) +class BaseInstaller(object): + def __init__(self, conf, log): + self.conf = conf + self.log = log + + @abc.abstractproperty + def node_user_name(self): + """user name for login to cloud node""" + + @abc.abstractmethod + def get_ssh_key_from_installer(self): + pass + + @abc.abstractmethod + def setup(self): + pass + + @abc.abstractmethod + def cleanup(self): + pass diff --git a/tests/installer/common/congress.py b/tests/installer/common/congress.py new file mode 100644 index 00000000..db882de2 --- /dev/null +++ b/tests/installer/common/congress.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +def set_doctor_driver_conf(ssh_client, restart_cmd): + cg_set_cmd = '''#!/bin/bash +co_conf=/etc/congress/congress.conf +co_conf_bak=/etc/congress/congress.conf.bak +co_entry="congress.datasources.doctor_driver.DoctorDriver" +if sudo grep -e "^drivers.*$co_entry" $co_conf; then + echo "NOTE: congress is configured as we needed" +else + echo "modify the congress config" + sudo cp $co_conf $co_conf_bak + sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf + %s +fi + ''' % (restart_cmd) + + ret, output = ssh_client.ssh(cg_set_cmd) + if ret: + raise Exception('Do the congress command in controller node failed....' + 'ret=%s, cmd=%s, output=%s' % (ret, cg_set_cmd, output)) + + +def restore_doctor_driver_conf(ssh_client, restart_cmd): + cg_restore_cmd = '''#!/bin/bash +co_conf=/etc/congress/congress.conf +co_conf_bak=/etc/congress/congress.conf.bak +if [ -e $co_conf_bak ]; then + echo "restore the congress config" + sudo cp $co_conf_bak $co_conf + sudo rm $co_conf_bak + %s +else + echo "Do not need to restore the congress config" +fi + ''' % (restart_cmd) + + ret, output = ssh_client.ssh(cg_restore_cmd) + if ret: + raise Exception('Do the congress command in controller node failed....' + 'ret=%s, cmd=%s, output=%s' % (ret, cg_restore_cmd, output)) diff --git a/tests/installer/common/restore_ceilometer.py b/tests/installer/common/restore_ceilometer.py new file mode 100644 index 00000000..d25b9ede --- /dev/null +++ b/tests/installer/common/restore_ceilometer.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import shutil + +ep_file = '/etc/ceilometer/event_pipeline.yaml' +ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak' + + +def restore_ep_config(): + + if not os.path.isfile(ep_file_bak): + print('Bak_file:%s does not exist.' % ep_file_bak) + else: + print('restore') + shutil.copyfile(ep_file_bak, ep_file) + os.remove(ep_file_bak) + return + + +restore_ep_config() diff --git a/tests/installer/common/set_ceilometer.py b/tests/installer/common/set_ceilometer.py new file mode 100644 index 00000000..f5946cb2 --- /dev/null +++ b/tests/installer/common/set_ceilometer.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import shutil +import yaml + +ep_file = '/etc/ceilometer/event_pipeline.yaml' +ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak' +event_notifier_topic = 'notifier://?topic=alarm.all' + + +def set_notifier_topic(): + config_modified = False + + if not os.path.isfile(ep_file): + raise Exception("File doesn't exist: %s." % ep_file) + + with open(ep_file, 'r') as file: + config = yaml.safe_load(file) + + sinks = config['sinks'] + for sink in sinks: + if sink['name'] == 'event_sink': + publishers = sink['publishers'] + if event_notifier_topic not in publishers: + print('Add event notifier in ceilometer') + publishers.append(event_notifier_topic) + config_modified = True + else: + print('NOTE: event notifier is configured in ceilometer as we needed') + + if config_modified: + shutil.copyfile(ep_file, ep_file_bak) + with open(ep_file, 'w+') as file: + file.write(yaml.safe_dump(config)) + + +set_notifier_topic() diff --git a/tests/installer/local.py b/tests/installer/local.py new file mode 100644 index 00000000..abe0ba25 --- /dev/null +++ b/tests/installer/local.py @@ -0,0 +1,97 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import shutil + +from installer.base import BaseInstaller +from utils import load_json_file +from utils import write_json_file + + +class LocalInstaller(BaseInstaller): + node_user_name = 'root' + + nova_policy_file = '/etc/nova/policy.json' + nova_policy_file_backup = '%s%s' % (nova_policy_file, '.bak') + + def __init__(self, conf, log): + super(LocalInstaller, self).__init__(conf, log) + self.policy_modified = False + self.add_policy_file = False + + def setup(self): + self.get_ssh_key_from_installer() + self.set_apply_patches() + + def cleanup(self): + self.restore_apply_patches() + + def get_ssh_key_from_installer(self): + self.log.info('Assuming SSH keys already exchanged with computer for local installer type') + return + + def set_apply_patches(self): + self._set_nova_policy() + + def restore_apply_patches(self): + self._restore_nova_policy() + + def _set_nova_policy(self): + host_status_policy = 'os_compute_api:servers:show:host_status' + host_status_rule = 'rule:admin_or_owner' + policy_data = { + 'context_is_admin': 'role:admin', + 'owner': 'user_id:%(user_id)s', + 'admin_or_owner': 'rule:context_is_admin or rule:owner', + host_status_policy: host_status_rule + } + + if os.path.isfile(self.nova_policy_file): + data = load_json_file(self.nova_policy_file) + if host_status_policy in data: + rule_origion = data[host_status_policy] + if host_status_rule == rule_origion: + self.log.info('Do not need to modify nova policy.') + self.policy_modified = False + else: + # update the host_status_policy + data[host_status_policy] = host_status_rule + self.policy_modified = True + else: + # add the host_status_policy, if the admin_or_owner is not + # defined, add it also + for policy, rule in policy_data.items(): + if policy not in data: + data[policy] = rule + self.policy_modified = True + if self.policy_modified: + self.log.info('Nova policy is Modified.') + shutil.copyfile(self.nova_policy_file, + self.nova_policy_file_backup) + else: + # file does not exit, create a new one and add the policy + self.log.info('Nova policy file not exist. Creating a new one') + data = policy_data + self.add_policy_file = True + + if self.policy_modified or self.add_policy_file: + write_json_file(self.nova_policy_file, data) + os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"') + + def _restore_nova_policy(self): + if self.policy_modified: + shutil.copyfile(self.nova_policy_file_backup, self.nova_policy_file) + os.remove(self.nova_policy_file_backup) + elif self.add_policy_file: + os.remove(self.nova_policy_file) + + if self.add_policy_file or self.policy_modified: + os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"') + self.add_policy_file = False + self.policy_modified = False diff --git a/tests/instance.py b/tests/instance.py new file mode 100644 index 00000000..c6acbc3d --- /dev/null +++ b/tests/instance.py @@ -0,0 +1,114 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import time + +from oslo_config import cfg + +from identity_auth import get_identity_auth +from identity_auth import get_session +from os_clients import neutron_client +from os_clients import nova_client + +OPTS = [ + cfg.StrOpt('flavor', + default='m1.tiny', + help='the name of flavor', + required=True), + cfg.IntOpt('instance_count', + default=os.environ.get('VM_COUNT', 1), + help='the count of instance', + required=True), + cfg.StrOpt('instance_basename', + default='doctor_vm', + help='the base name of instance', + required=True), +] + + +class Instance(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.auth = get_identity_auth(username=self.conf.doctor_user, + password=self.conf.doctor_passwd, + project=self.conf.doctor_project) + self.nova = \ + nova_client(conf.nova_version, + get_session(auth=self.auth)) + self.neutron = neutron_client(get_session(auth=self.auth)) + self.servers = {} + self.vm_names = [] + + def create(self): + self.log.info('instance create start......') + + # get flavor, image and network for vm boot + flavors = {flavor.name: flavor for flavor in self.nova.flavors.list()} + flavor = flavors.get(self.conf.flavor) + image = self.nova.glance.find_image(self.conf.image_name) + network = self.neutron.list_networks(name=self.conf.net_name)['networks'][0] + nics = {'net-id': network['id']} + + self.servers = \ + {getattr(server, 'name'): server + for server in self.nova.servers.list()} + for i in range(0, self.conf.instance_count): + vm_name = "%s%d"%(self.conf.instance_basename, i) + self.vm_names.append(vm_name) + if vm_name not in self.servers: + server = self.nova.servers.create(vm_name, image, + flavor, nics=[nics]) + self.servers[vm_name] = server + time.sleep(0.1) + + self.log.info('instance create end......') + + def delete(self): + self.log.info('instance delete start.......') + + for vm_name in self.vm_names: + if vm_name in self.servers: + self.nova.servers.delete(self.servers[vm_name]) + time.sleep(0.1) + + # check that all vms are deleted + while self.nova.servers.list(): + time.sleep(0.1) + self.servers.clear() + del self.vm_names[:] + + self.log.info('instance delete end.......') + + def wait_for_vm_launch(self): + self.log.info('wait for vm launch start......') + + wait_time = 60 + count = 0 + while count < wait_time: + active_count = 0 + for vm_name in self.vm_names: + server = self.nova.servers.get(self.servers[vm_name]) + server_status = getattr(server, 'status').lower() + if 'active' == server_status: + active_count += 1 + elif 'error' == server_status: + raise Exception('vm launched with error state') + else: + time.sleep(2) + count += 1 + continue + if active_count == self.conf.instance_count: + self.log.info('wait for vm launch end......') + return + count += 1 + time.sleep(2) + raise Exception('time out for vm launch') + diff --git a/tests/lib/installers/apex b/tests/lib/installers/apex index 55878dec..f7b9624e 100644 --- a/tests/lib/installers/apex +++ b/tests/lib/installers/apex @@ -5,7 +5,7 @@ ssh_opts_cpu="$ssh_opts -i instack_key" function get_installer_ip { is_set INSTALLER_IP && return - INSTALLER_IP=$(get_first_vnic_ip instack) + INSTALLER_IP=$(get_first_vnic_ip undercloud) } function installer_get_ssh_keys { @@ -44,13 +44,14 @@ function installer_apply_patches { fi co_conf=/etc/congress/congress.conf + co_conf_bak=/etc/congress/congress.conf.bak co_entry="congress.datasources.doctor_driver.DoctorDriver" if sudo grep -e "^drivers.*$co_entry" $co_conf; then echo "NOTE: congress is configured as we needed" else echo "modify the congress config" - sudo sed -i -e "/^drivers/s/$/,$co_entry # added by doctor script/" \ - $co_conf + sudo cp $co_conf $co_conf_bak + sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf sudo systemctl restart openstack-congress-server.service fi ' > installer_apply_patches_$node.log 2>&1 @@ -90,10 +91,10 @@ function installer_revert_patches { date co_conf=/etc/congress/congress.conf - co_entry="congress.datasources.doctor_driver.DoctorDriver" - if sudo grep -q -e "# added by doctor script" $co_conf; then - echo "modify the congress config" - sudo sed -i -e "/^drivers/s/^\(.*\),$co_entry # added by doctor script/\1/" $co_conf + co_conf_bak=/etc/congress/congress.conf.bak + if [ -e $co_conf_bak ]; then + echo "restore the congress config" + sudo mv $co_conf_bak $co_conf sudo systemctl restart openstack-congress-server.service fi diff --git a/tests/lib/installers/fuel b/tests/lib/installers/fuel index 0c56963c..85865720 100644 --- a/tests/lib/installers/fuel +++ b/tests/lib/installers/fuel @@ -96,8 +96,15 @@ function installer_apply_patches { service nova-api restart fi else - # TODO(tojuvone) policy.json might not exists in Ocata. - echo "$np_conf does not exist!!!" + # policy.json does not exist in Ocata. + echo "$np_conf does not exist. Creating new one." + echo -e "{\n \"context_is_admin\": \"role:admin\"," > $np_conf + echo -e " \"owner\" : \"user_id:%(user_id)s\"," >> $np_conf + echo -e " \"admin_or_owner\": \"rule:context_is_admin or rule:owner\"," >> $np_conf + echo -e " \"os_compute_api:servers:show:host_status\": \"rule:admin_or_owner\" \n}" >> $np_conf + np_rm="${np_conf}-doctor-rm" + cp $np_conf $np_rm + service nova-api restart fi ' > installer_apply_patches_$node.log 2>&1 done @@ -167,14 +174,16 @@ function installer_revert_patches { fi np_conf=/etc/nova/policy.json - entry="os_compute_api:servers:show:host_status" - if [ -e $np_conf ]; then - np_backup="${np_conf}-doctor-saved" - if [ -e $np_backup ]; then - cp -f $np_backup $np_conf - rm $np_backup - service nova-api restart - fi + np_backup="${np_conf}-doctor-saved" + np_rm="${np_conf}-doctor-rm" + if [ -e $np_backup ]; then + cp -f $np_backup $np_conf + rm $np_backup + service nova-api restart + elif [ -e $np_rm ]; then + rm $np_conf + rm $np_rm + service nova-api restart fi ' >> installer_apply_patches_$node.log 2>&1 done diff --git a/tests/lib/installers/local b/tests/lib/installers/local index 50c3686f..d628867a 100644 --- a/tests/lib/installers/local +++ b/tests/lib/installers/local @@ -9,7 +9,42 @@ function installer_get_ssh_keys { } function installer_apply_patches { - # Noop + set -x + date + echo "### apply patches (installer=local)" + np_conf=/etc/nova/policy.json + if [ -e $np_conf ]; then + entry="os_compute_api:servers:show:host_status" + new="rule:admin_or_owner" + np_backup="${np_conf}-doctor-saved" + if grep -q "${entry}.*${new}" $np_conf; then + echo "Not modifying nova policy" + elif grep -q "${entry}" $np_conf; then + echo "modify nova policy" + cp $np_conf $np_backup + oldline=$(grep "$entry" $np_conf) + newline=$(echo "$oldline" | sed "s/rule.*\"/$new\"/") + sed -i "s/$oldline/$newline/" $np_conf + # TODO(umar): Update to systemd when screen is no more used for devstack + screen -S stack -p n-api -X stuff "^C^M^[[A^M" # restart n-api service + else + echo "add nova policy" + cp $np_conf $np_backup + sed -i "/{/a \ \"${entry}\": \"$new\"" $np_conf + screen -S stack -p n-api -X stuff "^C^M^[[A^M" + fi + else + # policy.json does not exist in Ocata. + echo "$np_conf does not exist. Creating a new one" + echo -e '{\n "context_is_admin": "role:admin",' > $np_conf + echo -e ' "owner" : "user_id:%(user_id)s",' >> $np_conf + echo -e ' "admin_or_owner": "rule:context_is_admin or rule:owner",' >> $np_conf + echo -e ' "os_compute_api:servers:show:host_status": "rule:admin_or_owner"\n}' >> $np_conf + np_rm="${np_conf}-doctor-rm" + cp $np_conf $np_rm + screen -S stack -p n-api -X stuff "^C^M^[[A^M" + fi + return } @@ -31,6 +66,22 @@ function get_compute_ip_from_hostname { } function cleanup_installer { - # Noop + set -x + echo "### revert patches (installer=local)" + date + + np_conf=/etc/nova/policy.json + np_backup="${np_conf}-doctor-saved" + np_rm="${np_conf}-doctor-rm" + if [ -e $np_backup ]; then + cp -f $np_backup $np_conf + rm $np_backup + screen -S stack -p n-api -X stuff "^C^M^[[A^M" + elif [ -e $np_rm ]; then + rm $np_conf + rm $np_rm + screen -S stack -p n-api -X stuff "^C^M^[[A^M" + fi + return } diff --git a/tests/lib/monitor b/tests/lib/monitor new file mode 100644 index 00000000..6b804ec2 --- /dev/null +++ b/tests/lib/monitor @@ -0,0 +1,31 @@ +#!/bin/bash + +MONITOR_TYPE=${MONITOR_TYPE:-sample} + +function is_monitor_supported { + local monitor="$1" + [[ -f $TOP_DIR/lib/monitors/$monitor/$monitor ]] +} + +function is_monitor { + local monitor="$1" + [[ $monitor == $MONITOR_TYPE ]] +} + +function start_monitor { + start_monitor_$MONITOR_TYPE +} + +function stop_monitor { + stop_monitor_$MONITOR_TYPE +} + +function cleanup_monitor { + cleanup_monitor_$MONITOR_TYPE +} + +if ! is_monitor_supported $MONITOR_TYPE; then + die $LINENO "MONITOR_TYPE=$MONITOR_TYPE is not supported." +fi + +source $TOP_DIR/lib/monitors/$MONITOR_TYPE/$MONITOR_TYPE diff --git a/tests/lib/monitors/collectd/collectd b/tests/lib/monitors/collectd/collectd new file mode 100644 index 00000000..f5096658 --- /dev/null +++ b/tests/lib/monitors/collectd/collectd @@ -0,0 +1,101 @@ +#!/bin/bash + +function start_monitor_collectd { + ## CONTROL_IP is the IP of primary interface of control node i.e. + ## eth0, eno1. It is used by collectd monitor to communicate with + ## sample inspector. + ## @TODO (umar) see if mgmt IP of control is a better option. Also + ## primary interface may not be the right option + CONTROL_IP="$(ip a | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | sed -n 1p)" + #CONTROL_IP=192.168.98.6 + + echo " +Hostname \"$COMPUTE_HOST\" +FQDNLookup false +Interval 1 +MaxReadInterval 2 + +<LoadPlugin python> + Globals true +</LoadPlugin> +LoadPlugin ovs_events +LoadPlugin logfile + +<Plugin logfile> + File \"/var/log/collectd.log\" + Timestamp true + LogLevel \"info\" +</Plugin> + +<Plugin python> + ModulePath \"/home/$COMPUTE_USER\" + LogTraces true + Interactive false + Import \"collectd_plugin\" + <Module \"collectd_plugin\"> + control_ip \"$CONTROL_IP\" + compute_ip \"$COMPUTE_IP\" + compute_host \"$COMPUTE_HOST\" + compute_user \"$COMPUTE_USER\" + inspector_type \"$INSPECTOR_TYPE\" + os_auth_url \"$OS_AUTH_URL\" + os_username \"$OS_USERNAME\" + os_password \"$OS_PASSWORD\" + os_project_name \"$OS_PROJECT_NAME\" + os_user_domain_name \"$OS_USER_DOMAIN_NAME\" + os_user_domain_id \"$OS_USER_DOMAIN_ID\" + os_project_domain_name \"$OS_PROJECT_DOMAIN_NAME\" + os_project_domain_id \"$OS_PROJECT_DOMAIN_ID\" + </Module> +</Plugin> + +<Plugin ovs_events> + Port 6640 + Socket \"/var/run/openvswitch/db.sock\" + Interfaces \"@INTERFACE_NAME@\" + SendNotification true + DispatchValues false +</Plugin> + +" > $TOP_DIR/lib/monitors/collectd.conf + + scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd.conf $COMPUTE_USER@$COMPUTE_IP: + ## @TODO (umar) Always assuming that the interface is assigned an IP if + ## interface name is not provided. See if there is a better approach + ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" " + if [ -n \"$INTERFACE_NAME\" ]; then + dev=$INTERFACE_NAME + else + dev=\$(sudo ip a | awk '/ $COMPUTE_IP\//{print \$NF}') + fi + sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf + collectd_conf=/opt/collectd/etc/collectd.conf + if [ -e \$collectd_conf ]; then + sudo cp \$collectd_conf \${collectd_conf}-doctor-saved + else + sudo touch \${collectd_conf}-doctor-created + fi + sudo mv collectd.conf /opt/collectd/etc/collectd.conf" + + scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd/collectd_plugin.py $COMPUTE_USER@$COMPUTE_IP:collectd_plugin.py + ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "sudo pkill collectd + sudo /opt/collectd/sbin/collectd" +} + +function stop_monitor_collectd { + ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'sudo pkill collectd' +} + +function cleanup_monitor_collectd { + ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" " + collectd_conf=/opt/collectd/etc/collectd.conf + if [ -e \"\${collectd_conf}-doctor-created\" ]; then + sudo rm \"\${collectd_conf}-doctor-created\" + sudo rm \$collectd_conf + elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then + sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf + sudo rm \"\${collectd_conf}-doctor-saved\" + fi" + + rm $TOP_DIR/lib/monitors/collectd.conf +} diff --git a/tests/lib/monitors/collectd/collectd_plugin.py b/tests/lib/monitors/collectd/collectd_plugin.py new file mode 100644 index 00000000..70fcf26e --- /dev/null +++ b/tests/lib/monitors/collectd/collectd_plugin.py @@ -0,0 +1,167 @@ +############################################################################## +# Copyright (c) 2017 NEC Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import collectd +import sys +from netifaces import interfaces, ifaddresses, AF_INET +from datetime import datetime +import json +import requests +import time +from requests.exceptions import ConnectionError + +from keystoneauth1 import loading +from keystoneauth1 import session +from congressclient.v1 import client + + +def write_debug(str_write, write_type, compute_user): + file_name = ('/home/%s/monitor.log' % compute_user) + file_tmp = open(file_name, write_type) + file_tmp.write( "%s" % str_write) + file_tmp.close() + + +class DoctorMonitorCollectd(object): + def __init__(self): + self.control_ip = '' + self.compute_user = '' + self.compute_ip = '' + self.host_name = '' + self.inspector_type = '' + self.inspector_url = '' + self.os_auth_url = '' + self.os_username = '' + self.os_password = '' + self.os_project_name = '' + self.os_user_domain_name = '' + self.os_user_domain_id = '' + self.os_project_domain_name = '' + self.os_project_domain_id = '' + self.sess = '' + self.auth = '' + self.inspector_notified = 0 + self.start_notifications = 0 + self.monitor_type = 'sample' + + def config_func(self, config): + for node in config.children: + key = node.key.lower() + val = node.values[0] + + if key == 'compute_host': + self.host_name = val + elif key == 'control_ip': + self.control_ip = val + elif key == 'compute_ip': + self.compute_ip = val + elif key == 'compute_user': + self.compute_user = val + elif key == 'inspector_type': + self.inspector_type = val + elif key == 'os_auth_url': + self.os_auth_url = val + elif key == 'os_username': + self.os_username = val + elif key == 'os_password': + self.os_password = val + elif key == 'os_project_name': + self.os_project_name = val + elif key == 'os_user_domain_name': + self.os_user_domain_name = val + elif key == 'os_user_domain_id': + self.os_user_domain_id = val + elif key == 'os_project_domain_name': + self.os_project_domain_name = val + elif key == 'os_project_domain_id': + self.os_project_domain_id = val + else: + collectd.info('Unknown config key "%s"' % key) + + def init_collectd(self): + write_debug("Compute node collectd monitor start at %s\n\n" % datetime.now().isoformat(), "w", self.compute_user) + + if self.inspector_type == 'sample': + self.inspector_url = ('http://%s:12345/events' % self.control_ip) + elif self.inspector_type == 'congress': + loader = loading.get_plugin_loader('password') + self.auth = loader.load_from_options(auth_url=self.os_auth_url, + username=self.os_username, + password=self.os_password, + project_name=self.os_project_name, + user_domain_name=self.os_user_domain_name, + user_domain_id=self.os_user_domain_id, + project_domain_name=self.os_project_domain_name, + project_domain_id=self.os_project_domain_id) + self.sess=session.Session(auth=self.auth) + congress = client.Client(session=self.sess, service_type='policy') + ds = congress.list_datasources()['results'] + doctor_ds = next((item for item in ds if item['driver'] == 'doctor'), + None) + + congress_endpoint = congress.httpclient.get_endpoint(auth=self.auth) + self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' % + (congress_endpoint, doctor_ds['id'])) + else: + sys.exit() + self.start_notifications = 1 + + + def notify_inspector(self): + event_type = "compute.host.down" + payload = [ + { + 'id': ("monitor_%s_id1" % self.monitor_type), + 'time': datetime.now().isoformat(), + 'type': event_type, + 'details': { + 'hostname': self.host_name, + 'status': 'down', + 'monitor': ("monitor_%s" % self.monitor_type), + 'monitor_event_id': ("monitor_%s_event1" % self.monitor_type) + }, + }, + ] + data = json.dumps(payload) + self.inspector_notified = 1 + + if self.inspector_type == 'sample': + headers = {'content-type': 'application/json'} + try: + requests.post(self.inspector_url, data=data, headers=headers) + except ConnectionError as err: + print err + elif self.inspector_type == 'congress': + # TODO(umar) enhance for token expiry case + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'X-Auth-Token': self.sess.get_token() + } + requests.put(self.inspector_url, data=data, headers=headers) + + + def handle_notif(self, notification, data=None): + if (notification.severity == collectd.NOTIF_FAILURE or + notification.severity == collectd.NOTIF_WARNING): + if (self.start_notifications == 1 and self.inspector_notified == 0): + write_debug("Received down notification: doctor monitor detected at %s\n" % time.time(), "a", self.compute_user) + self.notify_inspector() + + elif notification.severity == collectd.NOTIF_OKAY: + collectd.info("Interface status: UP again %s\n" % time.time()) + else: + collectd.info("Unknown notification severity %s\n" % notification.severity) + + +monitor = DoctorMonitorCollectd() + +collectd.register_config(monitor.config_func) +collectd.register_init(monitor.init_collectd) +collectd.register_notification(monitor.handle_notif) diff --git a/tests/monitor.py b/tests/lib/monitors/sample/monitor.py index 7450c534..7450c534 100644 --- a/tests/monitor.py +++ b/tests/lib/monitors/sample/monitor.py diff --git a/tests/lib/monitors/sample/sample b/tests/lib/monitors/sample/sample new file mode 100644 index 00000000..1d310333 --- /dev/null +++ b/tests/lib/monitors/sample/sample @@ -0,0 +1,18 @@ +#!/bin/bash + +function start_monitor_sample { + cp $TOP_DIR/lib/monitors/sample/monitor.py $TOP_DIR/monitor.py + pgrep -f "python monitor.py" && return 0 + sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \ + > monitor.log 2>&1 & +} + +function stop_monitor_sample { + pgrep -f "python monitor.py" || return 0 + sudo kill $(pgrep -f "python monitor.py") +} + +function cleanup_monitor_sample { + rm monitor.py + return +} diff --git a/tests/logger.py b/tests/logger.py index 72043ab3..021389d9 100644 --- a/tests/logger.py +++ b/tests/logger.py @@ -21,6 +21,7 @@ class Logger(object): CI_DEBUG = os.getenv('CI_DEBUG') + logging.basicConfig(filemode='w') self.logger = logging.getLogger(logger_name) self.logger.propagate = 0 self.logger.setLevel(logging.DEBUG) @@ -41,7 +42,5 @@ class Logger(object): file_handler.setLevel(logging.DEBUG) self.logger.addHandler(file_handler) - def getLogger(self): return self.logger - diff --git a/tests/main.py b/tests/main.py index 50e0821b..db2fafd9 100644 --- a/tests/main.py +++ b/tests/main.py @@ -6,11 +6,21 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import os +from os.path import isfile, join import sys +from alarm import Alarm import config +from consumer import get_consumer from image import Image +from instance import Instance +from inspector import get_inspector +from installer import get_installer import logger as doctor_log +from user import User +from network import Network +from monitor import get_monitor LOG = doctor_log.Logger('doctor').getLogger() @@ -20,38 +30,82 @@ class DoctorTest(object): def __init__(self, conf): self.conf = conf - self.image = Image(self.conf) + self.image = Image(self.conf, LOG) + self.user = User(self.conf, LOG) + self.network = Network(self.conf, LOG) + self.instance = Instance(self.conf, LOG) + self.alarm = Alarm(self.conf, LOG) + self.inspector = get_inspector(self.conf, LOG) + self.monitor = get_monitor(self.conf, + self.inspector.get_inspector_url(), + LOG) + self.consumer = get_consumer(self.conf, LOG) + self.installer = get_installer(self.conf, LOG) + + def setup(self): + # prepare the cloud env + self.installer.setup() + + # preparing VM image... + self.image.create() + + # creating test user... + self.user.create() + self.user.update_quota() + + # creating VM... + self.network.create() + self.instance.create() + self.instance.wait_for_vm_launch() + + # creating alarm... + self.alarm.create() + + # starting doctor sample components... + self.inspector.start() + self.monitor.start() + self.consumer.start() def run(self): """run doctor test""" try: LOG.info('doctor test starting.......') - # prepare the cloud env - # preparing VM image... - self.image.create() - - # creating test user... - - # creating VM... - - # creating alarm... - - # starting doctor sample components... + self.setup() # injecting host failure... + # NOTE (umar) add INTERFACE_NAME logic to host injection # verify the test results + # NOTE (umar) copy remote monitor.log file when monitor=collectd + except Exception as e: LOG.error('doctor test failed, Exception=%s' % e) sys.exit(1) finally: - self.image.delete() + self.cleanup() + + def cleanup(self): + self.alarm.delete() + self.instance.delete() + self.network.delete() + self.image.delete() + self.inspector.stop() + self.user.delete() + self.monitor.stop() + self.consumer.stop() + self.installer.cleanup() def main(): """doctor main""" - conf = config.prepare_conf() + doctor_root_dir = os.path.dirname(sys.path[0]) + config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/') + config_files = [join(config_file_dir, f) for f in os.listdir(config_file_dir) + if isfile(join(config_file_dir, f))] + + conf = config.prepare_conf(args=sys.argv[1:], + config_files=config_files) doctor = DoctorTest(conf) doctor.run() diff --git a/tests/monitor/__init__.py b/tests/monitor/__init__.py new file mode 100644 index 00000000..e268907f --- /dev/null +++ b/tests/monitor/__init__.py @@ -0,0 +1,29 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from oslo_config import cfg +from oslo_utils import importutils + +OPTS = [ + cfg.StrOpt('type', + default='sample', + choices=['sample', 'collectd'], + help='the type of doctor monitor component', + required=True), +] + + +_monitor_name_class_mapping = { + 'sample': 'monitor.sample.SampleMonitor', + 'collectd': 'monitor.collectd.CollectdMonitor' +} + +def get_monitor(conf, inspector_url, log): + monitor_class = _monitor_name_class_mapping.get(conf.monitor.type) + return importutils.import_object(monitor_class, conf, + inspector_url, log) diff --git a/tests/monitor/base.py b/tests/monitor/base.py new file mode 100644 index 00000000..ccb647cf --- /dev/null +++ b/tests/monitor/base.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseMonitor(object): + """Monitor computer fault and report error to the inspector""" + def __init__(self, conf, inspector_url, log): + self.conf = conf + self.log = log + self.inspector_url = inspector_url + + @abc.abstractmethod + def start(self): + pass + + @abc.abstractmethod + def stop(self): + pass diff --git a/tests/monitor/collectd.py b/tests/monitor/collectd.py new file mode 100644 index 00000000..f7a4f442 --- /dev/null +++ b/tests/monitor/collectd.py @@ -0,0 +1,145 @@ +############################################################################## +# Copyright (c) 2017 NEC Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os +import socket +import getpass +import sys + +from identity_auth import get_session +from os_clients import nova_client +from monitor.base import BaseMonitor + + +class CollectdMonitor(BaseMonitor): + def __init__(self, conf, inspector_url, log): + super(CollectdMonitor, self).__init__(conf, inspector_url, log) + self.top_dir = os.path.dirname(sys.path[0]) + self.session = get_session() + self.nova = nova_client(conf.nova_version, self.session) + self.compute_hosts = self.nova.hypervisors.list(detailed=True) + for host in self.compute_hosts: + host_dict = host.__dict__ + self.compute_host = host_dict['hypervisor_hostname'] + self.compute_ip = host_dict['host_ip'] + tmp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + tmp_sock.connect(("8.8.8.8", 80)) + + ## control_ip is the IP of primary interface of control node i.e. + ## eth0, eno1. It is used by collectd monitor to communicate with + ## sample inspector. + ## TODO (umar) see if mgmt IP of control is a better option. Also + ## primary interface may not be the right option + self.control_ip = tmp_sock.getsockname()[0] + self.compute_user = getpass.getuser() + self.interface_name = os.environ.get('INTERFACE_NAME') or '' + self.inspector_type = os.environ.get('INSPECTOR_TYPE', 'sample') + self.auth_url = os.environ.get('OS_AUTH_URL') + self.username = os.environ.get('OS_USERNAME') + self.password = os.environ.get('OS_PASSWORD') + self.project_name = os.environ.get('OS_PROJECT_NAME') + self.user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default' + self.user_domain_id = os.environ.get('OS_USER_DOMAIN_ID') + self.project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default' + self.project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID') + self.ssh_opts_cpu = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + + def start(self): + self.log.info("Collectd monitor start.........") + f = open("%s/tests/collectd.conf" % self.top_dir, 'w') + collectd_conf_file = """ +Hostname %s +FQDNLookup false +Interval 1 +MaxReadInterval 2 + +<LoadPlugin python> +Globals true +</LoadPlugin> +LoadPlugin ovs_events +LoadPlugin logfile + +<Plugin logfile> + File \"/var/log/collectd.log\" + Timestamp true + LogLevel \"info\" +</Plugin> + +<Plugin python> + ModulePath \"/home/%s\" + LogTraces true + Interactive false + Import \"collectd_plugin\" + <Module \"collectd_plugin\"> + control_ip \"%s\" + compute_ip \"%s\" + compute_host \"%s\" + compute_user \"%s\" + inspector_type \"%s\" + os_auth_url \"%s\" + os_username \"%s\" + os_password \"%s\" + os_project_name \"%s\" + os_user_domain_name \"%s\" + os_user_domain_id \"%s\" + os_project_domain_name \"%s\" + os_project_domain_id \"%s\" + </Module> +</Plugin> + +<Plugin ovs_events> + Port 6640 + Socket \"/var/run/openvswitch/db.sock\" + Interfaces \"@INTERFACE_NAME@\" + SendNotification true + DispatchValues false +</Plugin> + """ % (self.compute_host, self.compute_user, self.control_ip, self.compute_ip, self.compute_host, self.compute_user, + self.inspector_type, self.auth_url, self.username, self.password, self.project_name, self.user_domain_name, + self.user_domain_id, self.project_domain_name, self.project_domain_id) + f.write(collectd_conf_file) + f.close() + + os.system(" scp %s %s/tests/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip)) + self.log.info("after first scp") + ## @TODO (umar) Always assuming that the interface is assigned an IP if + ## interface name is not provided. See if there is a better approach + os.system(""" ssh %s %s@%s \"if [ -n \"%s\" ]; then + dev=%s + else + dev=\$(sudo ip a | awk '/ %s\//{print \$NF}') + fi + sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf + collectd_conf=/opt/collectd/etc/collectd.conf + if [ -e \$collectd_conf ]; then + sudo cp \$collectd_conf \${collectd_conf}-doctor-saved + else + sudo touch \${collectd_conf}-doctor-created + fi + sudo mv collectd.conf /opt/collectd/etc/collectd.conf\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip, self.interface_name, self.interface_name, self.compute_ip)) + self.log.info("after first ssh") + os.system(" scp %s %s/tests/lib/monitors/collectd/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip)) + self.log.info("after sec scp") + os.system(" ssh %s %s@%s \"sudo pkill collectd; sudo /opt/collectd/sbin/collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip)) + self.log.info("after sec ssh") + + def stop(self): + os.system(" ssh %s %s@%s \"sudo pkill collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip)) + + def cleanup(self): + os.system(""" ssh %s %s@%s \" + collectd_conf=/opt/collectd/etc/collectd.conf + if [ -e \"\${collectd_conf}-doctor-created\" ]; then + sudo rm \"\${collectd_conf}-doctor-created\" + sudo rm \$collectd_conf + elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then + sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf + sudo rm \"\${collectd_conf}-doctor-saved\" + fi\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip)) + os.remove("%s/tests/collectd.conf" % self.top_dir) diff --git a/tests/monitor/sample.py b/tests/monitor/sample.py new file mode 100644 index 00000000..1333a2ec --- /dev/null +++ b/tests/monitor/sample.py @@ -0,0 +1,115 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from datetime import datetime +import json +import requests +import socket +from threading import Thread +import time + +from identity_auth import get_session +from os_clients import nova_client +from monitor.base import BaseMonitor + + +class SampleMonitor(BaseMonitor): + event_type = "compute.host.down" + + def __init__(self, conf, inspector_url, log): + super(SampleMonitor, self).__init__(conf, inspector_url, log) + self.session = get_session() + self.nova = nova_client(conf.nova_version, self.session) + self.hosts = self.nova.hypervisors.list(detailed=True) + self.pingers = [] + + def start(self): + self.log.info('sample monitor start......') + for host in self.hosts: + host_dict = host.__dict__ + host_name = host_dict['hypervisor_hostname'] + host_ip = host_dict['host_ip'] + pinger = Pinger(host_name, host_ip, self, self.log) + pinger.start() + self.pingers.append(pinger) + + def stop(self): + self.log.info('sample monitor stop......') + for pinger in self.pingers: + pinger.stop() + pinger.join() + del self.pingers + + def report_error(self, hostname): + self.log.info('sample monitor report error......') + data = [ + { + 'id': 'monitor_sample_id1', + 'time': datetime.now().isoformat(), + 'type': self.event_type, + 'details': { + 'hostname': hostname, + 'status': 'down', + 'monitor': 'monitor_sample', + 'monitor_event_id': 'monitor_sample_event1' + }, + }, + ] + + auth_token = self.session.get_token() if \ + self.conf.inspector.type != 'sample' else None + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'X-Auth-Token': auth_token, + } + + url = '%s%s' % (self.inspector_url, 'events') \ + if self.inspector_url.endswith('/') else \ + '%s%s' % (self.inspector_url, '/events') + requests.put(url, data=json.dumps(data), headers=headers) + + +class Pinger(Thread): + interval = 0.1 # second + timeout = 0.1 # second + ICMP_ECHO_MESSAGE = bytes([0x08, 0x00, 0xf7, 0xff, 0x00, 0x00, 0x00, 0x00]) + + def __init__(self, host_name, host_ip, monitor, log): + Thread.__init__(self) + self.monitor = monitor + self.hostname = host_name + self.ip_addr = host_ip or socket.gethostbyname(self.hostname) + self.log = log + self._stopped = False + + def run(self): + self.log.info("Starting Pinger host_name(%s), host_ip(%s)" + % (self.hostname, self.ip_addr)) + + sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_ICMP) + sock.settimeout(self.timeout) + while True: + if self._stopped: + return + try: + sock.sendto(self.ICMP_ECHO_MESSAGE, (self.ip_addr, 0)) + sock.recv(4096) + except socket.timeout: + self.log.info("doctor monitor detected at %s" % time.time()) + self.monitor.report_error(self.hostname) + self.log.info("ping timeout, quit monitoring...") + self._stopped = True + return + time.sleep(self.interval) + + def stop(self): + self.log.info("Stopping Pinger host_name(%s), host_ip(%s)" + % (self.hostname, self.ip_addr)) + self._stopped = True diff --git a/tests/network.py b/tests/network.py new file mode 100644 index 00000000..da7ad09d --- /dev/null +++ b/tests/network.py @@ -0,0 +1,68 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from oslo_config import cfg + +from identity_auth import get_identity_auth +from identity_auth import get_session +from os_clients import neutron_client + + +OPTS = [ + cfg.StrOpt('net_name', + default='doctor_net', + help='the name of test net', + required=True), + cfg.StrOpt('net_cidr', + default='192.168.168.0/24', + help='the cidr of test subnet', + required=True), +] + + +class Network(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.auth = get_identity_auth(username=self.conf.doctor_user, + password=self.conf.doctor_passwd, + project=self.conf.doctor_project) + self.neutron = neutron_client(get_session(auth=self.auth)) + self.net = None + self.subnet = None + + def create(self): + self.log.info('network create start.......') + net_name = self.conf.net_name + networks = self.neutron.list_networks(name=net_name)['networks'] + self.net = networks[0] if networks \ + else self.neutron.create_network( + {'network': {'name': net_name}})['network'] + self.log.info('network create end.......') + + self.log.info('subnet create start.......') + subnets = self.neutron.list_subnets(network_id=self.net['id'])['subnets'] + subnet_param = {'name': net_name, 'network_id': self.net['id'], + 'cidr': self.conf.net_cidr, 'ip_version': 4, + 'enable_dhcp': False} + self.subnet = subnets[0] if subnets \ + else self.neutron.create_subnet( + {'subnet': subnet_param})['subnet'] + self.log.info('subnet create end.......') + + def delete(self): + self.log.info('subnet delete start.......') + if self.subnet: + self.neutron.delete_subnet(self.subnet['id']) + self.log.info('subnet delete end.......') + + self.log.info('network delete start.......') + if self.net: + self.neutron.delete_network(self.net['id']) + self.log.info('network delete end.......') diff --git a/tests/os_clients.py b/tests/os_clients.py index 2eb406e0..44fa3aad 100644 --- a/tests/os_clients.py +++ b/tests/os_clients.py @@ -8,14 +8,43 @@ ##############################################################################
from oslo_config import cfg
+import aodhclient.client as aodhclient
+from congressclient.v1 import client as congressclient
import glanceclient.client as glanceclient
+from keystoneclient.v2_0 import client as ks_client
+from neutronclient.v2_0 import client as neutronclient
+import novaclient.client as novaclient
OPTS = [
cfg.StrOpt('glance_version', default='2', help='glance version'),
+ cfg.StrOpt('nova_version', default='2.34', help='Nova version'),
+ cfg.StrOpt('aodh_version', default='2', help='aodh version'),
]
def glance_client(version, session):
return glanceclient.Client(version=version,
session=session)
+
+
+def keystone_client(session):
+ return ks_client.Client(session=session)
+
+
+def nova_client(version, session):
+ return novaclient.Client(version=version,
+ session=session)
+
+
+def neutron_client(session):
+ return neutronclient.Client(session=session)
+
+
+def aodh_client(version, session):
+ return aodhclient.Client(version, session=session)
+
+
+def congress_client(session):
+ return congressclient.Client(session=session,
+ service_type='policy')
diff --git a/tests/run.sh b/tests/run.sh index fda1e753..dceb0614 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -29,7 +29,7 @@ DOCTOR_USER=doctor DOCTOR_PW=doctor DOCTOR_PROJECT=doctor DOCTOR_ROLE=_member_ -PROFILER_TYPE=${PROFILER_TYPE:-none} +PROFILER_TYPE=${PROFILER_TYPE:-poc} PYTHON_ENABLE=${PYTHON_ENABLE:-false} TOP_DIR=$(cd $(dirname "$0") && pwd) @@ -48,7 +48,7 @@ as_admin_user="--os-username admin --os-project-name $DOCTOR_PROJECT get_compute_host_info() { # get computer host info which first VM boot in as admin user COMPUTE_HOST=$(openstack $as_admin_user server show ${VM_BASENAME}1 | - grep "OS-EXT-SRV-ATTR:host" | awk '{ print $4 }') + grep "OS-EXT-SRV-ATTR:host " | awk '{ print $4 }') compute_host_in_undercloud=${COMPUTE_HOST%%.*} die_if_not_set $LINENO COMPUTE_HOST "Failed to get compute hostname" @@ -212,17 +212,6 @@ create_alarm() { done } -start_monitor() { - pgrep -f "python monitor.py" && return 0 - sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \ - > monitor.log 2>&1 & -} - -stop_monitor() { - pgrep -f "python monitor.py" || return 0 - sudo kill $(pgrep -f "python monitor.py") -} - start_consumer() { pgrep -f "python consumer.py" && return 0 python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 & @@ -294,8 +283,12 @@ inject_failure() { echo "disabling network of compute host [$COMPUTE_HOST] for 3 mins..." cat > disable_network.sh << 'END_TXT' #!/bin/bash -x -dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}') sleep 1 +if [ -n "@INTERFACE_NAME@" ]; then + dev=@INTERFACE_NAME@ +else + dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}') +fi sudo ip link set $dev down echo "doctor set link down at" $(date "+%s.%N") sleep 180 @@ -303,6 +296,7 @@ sudo ip link set $dev up sleep 1 END_TXT sed -i -e "s/@COMPUTE_IP@/$COMPUTE_IP/" disable_network.sh + sed -i -e "s/@INTERFACE_NAME@/$INTERFACE_NAME/" disable_network.sh chmod +x disable_network.sh scp $ssh_opts_cpu disable_network.sh "$COMPUTE_USER@$COMPUTE_IP:" ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &' @@ -327,8 +321,11 @@ calculate_notification_time() { wait_consumer 60 #keep 'at' as the last keyword just before the value, and #use regex to get value instead of the fixed column + if [ ! -f monitor.log ]; then + scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:monitor.log" . + fi detected=$(grep "doctor monitor detected at" monitor.log |\ - sed -e "s/^.* at //") + sed -e "s/^.* at //" | tail -1) notified=$(grep "doctor consumer notified at" consumer.log |\ sed -e "s/^.* at //" | tail -1) @@ -424,18 +421,18 @@ run_profiler() { export DOCTOR_PROFILER_T09=$(python -c \ "print(int(($notified-$relative_start)*1000))") - python profiler-poc.py >doctor_profiler.log 2>&1 + python profiler-poc.py > doctor_profiler.log 2>&1 fi } cleanup() { set +e echo "cleanup..." - stop_monitor stop_inspector stop_consumer unset_forced_down_hosts + stop_monitor collect_logs vms=$(openstack $as_doctor_user server list) @@ -467,6 +464,7 @@ cleanup() { cleanup_installer cleanup_inspector + cleanup_monitor # NOTE: Temporal log printer. for f in $(find . -name '*.log') @@ -478,9 +476,23 @@ cleanup() { done } +setup_python_packages() { + sudo pip install flask==0.10.1 + command -v openstack || sudo pip install python-openstackclient==2.3.0 + command -v ceilometer || sudo pip install python-ceilometerclient==2.6.2 + command -v congress || sudo pip install python-congressclient==1.5.0 +} + # Main process -if $PYTHON_ENABLE; then +if [[ $PYTHON_ENABLE == [Tt]rue ]]; then + which tox || sudo pip install tox + if [ -f /usr/bin/apt-get ]; then + sudo apt-get install -y python3-dev + elif [ -f /usr/bin/yum ] ; then + sudo yum install -y python3-devel + fi + cd $TOP_DIR echo "executing tox..." tox @@ -492,9 +504,14 @@ git log --oneline -1 || true # ignore even you don't have git installed trap cleanup EXIT +setup_python_packages + source $TOP_DIR/functions-common source $TOP_DIR/lib/installer source $TOP_DIR/lib/inspector +source $TOP_DIR/lib/monitor + +rm -f *.log setup_installer @@ -524,8 +541,8 @@ echo "injecting host failure..." inject_failure check_host_status "(DOWN|UNKNOWN)" 60 -calculate_notification_time unset_forced_down_hosts +calculate_notification_time collect_logs run_profiler diff --git a/tests/user.py b/tests/user.py new file mode 100644 index 00000000..b21bd1a8 --- /dev/null +++ b/tests/user.py @@ -0,0 +1,163 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os + +from oslo_config import cfg + +from identity_auth import get_session +from os_clients import keystone_client +from os_clients import nova_client + + +OPTS = [ + cfg.StrOpt('doctor_user', + default='doctor', + help='the name of test user', + required=True), + cfg.StrOpt('doctor_passwd', + default='doctor', + help='the password of test user', + required=True), + cfg.StrOpt('doctor_project', + default='doctor', + help='the name of test project', + required=True), + cfg.StrOpt('doctor_role', + default='_member_', + help='the role of test user', + required=True), + cfg.IntOpt('quota_instances', + default=os.environ.get('VM_COUNT', 1), + help='the quota of instances in test user', + required=True), + cfg.IntOpt('quota_cores', + default=os.environ.get('VM_COUNT', 1), + help='the quota of cores in test user', + required=True), +] + + +class User(object): + + def __init__(self, conf, log): + self.conf = conf + self.log = log + self.keystone = \ + keystone_client(get_session()) + self.nova = \ + nova_client(conf.nova_version, get_session()) + self.users = {} + self.projects = {} + self.roles = {} + self.roles_for_user = {} + self.roles_for_admin = {} + + def create(self): + """create test user, project and etc""" + self.log.info('user create start......') + + self._create_project() + self._create_user() + self._create_role() + self._add_user_role_in_project(is_admin=False) + self._add_user_role_in_project(is_admin=True) + + self.log.info('user create end......') + + def _create_project(self): + """create test project""" + self.projects = {project.name: project + for project in self.keystone.tenants.list()} + if self.conf.doctor_project not in self.projects: + test_project = \ + self.keystone.tenants.create(self.conf.doctor_project) + self.projects[test_project.name] = test_project + + def _create_user(self): + """create test user""" + project = self.projects.get(self.conf.doctor_project) + self.users = {user.name: user for user in self.keystone.users.list()} + if self.conf.doctor_user not in self.users: + test_user = self.keystone.users.create( + self.conf.doctor_user, + password=self.conf.doctor_passwd, + tenant_id=project.id) + self.users[test_user.name] = test_user + + def _create_role(self): + """create test role""" + self.roles = {role.name: role for role in self.keystone.roles.list()} + if self.conf.doctor_role not in self.roles: + test_role = self.keystone.roles.create(self.conf.doctor_role) + self.roles[test_role.name] = test_role + + def _add_user_role_in_project(self, is_admin=False): + """add test user with test role in test project""" + project = self.projects.get(self.conf.doctor_project) + + user_name = 'admin' if is_admin else self.conf.doctor_user + user = self.users.get(user_name) + + role_name = 'admin' if is_admin else self.conf.doctor_role + role = self.roles.get(role_name) + + roles_for_user = self.roles_for_admin \ + if is_admin else self.roles_for_user + + roles_for_user = \ + {role.name: role for role in + self.keystone.roles.roles_for_user(user, tenant=project)} + if role_name not in roles_for_user: + self.keystone.roles.add_user_role(user, role, tenant=project) + roles_for_user[role_name] = role + + def delete(self): + """delete the test user, project and role""" + self.log.info('user delete start......') + + project = self.projects.get(self.conf.doctor_project) + user = self.users.get(self.conf.doctor_user) + role = self.roles.get(self.conf.doctor_role) + + if project: + if 'admin' in self.roles_for_admin: + self.keystone.roles.remove_user_role( + self.users['admin'], + self.roles['admin'], + tenant=project) + + if user: + if role and self.conf.doctor_role in self.roles_for_user: + self.keystone.roles.remove_user_role( + user, role, tenant=project) + self.keystone.roles.delete(role) + self.keystone.users.delete(user) + + self.keystone.tenants.delete(project) + self.log.info('user delete end......') + + def update_quota(self): + self.log.info('user quota update start......') + project = self.projects.get(self.conf.doctor_project) + user = self.users.get(self.conf.doctor_user) + + if project and user: + self.quota = self.nova.quotas.get(project.id, + user_id=user.id) + if self.conf.quota_instances > self.quota.instances: + self.nova.quotas.update(project.id, + instances=self.conf.quota_instances, + user_id=user.id) + if self.conf.quota_cores > self.quota.cores: + self.nova.quotas.update(project.id, + cores=self.conf.quota_cores, + user_id=user.id) + self.log.info('user quota update end......') + else: + raise Exception('No project or role for update quota') diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..41e22353 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,78 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import json +import os +import paramiko + + +def load_json_file(full_path): + """Loads JSON from file + :param target_filename: + :return: + """ + if not os.path.isfile(full_path): + raise Exception('File(%s) does not exist' % full_path) + + with open(full_path, 'r') as file: + return json.load(file) + + +def write_json_file(full_path, data): + """write JSON from file + :param target_filename: + :return: + """ + + with open(full_path, 'w+') as file: + file.write(json.dumps(data)) + + +class SSHClient(object): + def __init__(self, ip, username, password=None, pkey=None, + key_filename=None, log=None, look_for_keys=False, + allow_agent=False): + self.client = paramiko.SSHClient() + self.client.load_system_host_keys() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.client.connect(ip, username=username, password=password, + pkey=pkey, key_filename=key_filename, + look_for_keys=look_for_keys, + allow_agent=allow_agent) + self.log = log + + def __del__(self): + self.client.close() + + def ssh(self, command): + if self.log: + self.log.debug("Executing: %s" % command) + stdin, stdout, stderr = self.client.exec_command(command) + ret = stdout.channel.recv_exit_status() + output = list() + for line in stdout.read().splitlines(): + output.append(line.decode('utf-8')) + if ret: + if self.log: + self.log.debug("*** FAILED to run command %s (%s)" % (command, ret)) + raise Exception( + "Unable to run \ncommand: %s\nret: %s" + % (command, ret)) + if self.log: + self.log.debug("*** SUCCESSFULLY run command %s" % command) + return ret, output + + def scp(self, source, dest, method='put'): + if self.log: + self.log.info("Copy %s -> %s" % (source, dest)) + ftp = self.client.open_sftp() + if method == 'put': + ftp.put(source, dest) + elif method == 'get': + ftp.get(source, dest) + ftp.close() @@ -1,6 +1,6 @@ [tox] minversion = 2.3.1 -envlist = verify +envlist = py34 skipsdist = True [testenv] @@ -20,7 +20,8 @@ passenv = PROFILER_TYPE PYTHON_ENABLE CI_DEBUG - -[testenv:verify] + INSTALLER_TYPE + INSTALLER_IP changedir = {toxinidir}/tests commands = python main.py + |