summaryrefslogtreecommitdiffstats
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rw-r--r--tests/alarm.py95
-rw-r--r--tests/config.py28
-rw-r--r--tests/consumer/__init__.py37
-rw-r--r--tests/consumer/base.py26
-rw-r--r--tests/consumer/sample.py71
-rw-r--r--tests/identity_auth.py9
-rw-r--r--tests/image.py45
-rw-r--r--tests/inspector.py6
-rw-r--r--tests/inspector/__init__.py40
-rw-r--r--tests/inspector/base.py30
-rw-r--r--tests/inspector/congress.py94
-rw-r--r--tests/inspector/sample.py151
-rw-r--r--tests/installer/__init__.py38
-rw-r--r--tests/installer/apex.py99
-rw-r--r--tests/installer/base.py32
-rw-r--r--tests/installer/common/congress.py47
-rw-r--r--tests/installer/common/restore_ceilometer.py27
-rw-r--r--tests/installer/common/set_ceilometer.py44
-rw-r--r--tests/installer/local.py97
-rw-r--r--tests/instance.py114
-rw-r--r--tests/lib/installers/apex15
-rw-r--r--tests/lib/installers/fuel29
-rw-r--r--tests/lib/installers/local55
-rw-r--r--tests/lib/monitor31
-rw-r--r--tests/lib/monitors/collectd/collectd101
-rw-r--r--tests/lib/monitors/collectd/collectd_plugin.py167
-rw-r--r--tests/lib/monitors/sample/monitor.py (renamed from tests/monitor.py)0
-rw-r--r--tests/lib/monitors/sample/sample18
-rw-r--r--tests/logger.py3
-rw-r--r--tests/main.py82
-rw-r--r--tests/monitor/__init__.py29
-rw-r--r--tests/monitor/base.py27
-rw-r--r--tests/monitor/collectd.py145
-rw-r--r--tests/monitor/sample.py115
-rw-r--r--tests/network.py68
-rw-r--r--tests/os_clients.py29
-rwxr-xr-xtests/run.sh55
-rw-r--r--tests/user.py163
-rw-r--r--tests/utils.py78
39 files changed, 2251 insertions, 89 deletions
diff --git a/tests/alarm.py b/tests/alarm.py
new file mode 100644
index 00000000..0582085e
--- /dev/null
+++ b/tests/alarm.py
@@ -0,0 +1,95 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import aodh_client
+from os_clients import nova_client
+
+OPTS = [
+ cfg.StrOpt('alarm_basename',
+ default='doctor_alarm',
+ help='the base name of alarm',
+ required=True),
+]
+
+
+class Alarm(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(username=self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ project=self.conf.doctor_project)
+ self.aodh = \
+ aodh_client(conf.aodh_version,
+ get_session(auth=self.auth))
+ self.nova = \
+ nova_client(conf.nova_version,
+ get_session(auth=self.auth))
+ self._init_alarm_name()
+
+ def _init_alarm_name(self):
+ self.alarm_names = []
+ for i in range(0, self.conf.instance_count):
+ alarm_name = '%s%d' % (self.conf.alarm_basename, i)
+ self.alarm_names.append(alarm_name)
+
+ def create(self):
+ self.log.info('alarm create start......')
+
+ alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()}
+ servers = \
+ {getattr(server, 'name'): server
+ for server in self.nova.servers.list()}
+
+ for i in range(0, self.conf.instance_count):
+ alarm_name = self.alarm_names[i]
+ if alarm_name in alarms:
+ continue;
+ vm_name = '%s%d' % (self.conf.instance_basename, i)
+ vm_id = getattr(servers[vm_name], 'id')
+ alarm_request = dict(
+ name=alarm_name,
+ description=u'VM failure',
+ enabled=True,
+ alarm_actions=[u'http://%s:%d/failure'
+ % (self.conf.consumer.ip,
+ self.conf.consumer.port)],
+ repeat_actions=False,
+ severity=u'moderate',
+ type=u'event',
+ event_rule=dict(
+ event_type=u'compute.instance.update',
+ query=[
+ dict(field=u'traits.instance_id',
+ type='',
+ op=u'eq',
+ value=vm_id),
+ dict(field=u'traits.state',
+ type='',
+ op=u'eq',
+ value=u'error')]))
+ self.aodh.alarm.create(alarm_request)
+
+ self.log.info('alarm create end......')
+
+ def delete(self):
+ self.log.info('alarm delete start.......')
+
+ alarms = {alarm['name']: alarm for alarm in self.aodh.alarm.list()}
+ for alarm_name in self.alarm_names:
+ if alarm_name in alarms:
+ self.aodh.alarm.delete(alarms[alarm_name]['alarm_id'])
+
+ del self.alarm_names[:]
+
+ self.log.info('alarm delete end.......')
diff --git a/tests/config.py b/tests/config.py
index 2288d36e..f33ab5d2 100644
--- a/tests/config.py
+++ b/tests/config.py
@@ -6,20 +6,39 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+import itertools
+
from oslo_config import cfg
+import alarm
+import consumer
import image
+import instance
+import installer
+import network
+import inspector
+import monitor
import os_clients
+import user
def list_opts():
return [
- ('os_clients', os_clients.OPTS),
- ('image', image.IMAGE_OPTS),
+ ('installer', installer.OPTS),
+ ('monitor', monitor.OPTS),
+ ('inspector', inspector.OPTS),
+ ('consumer', consumer.OPTS),
+ ('DEFAULT', itertools.chain(
+ os_clients.OPTS,
+ image.OPTS,
+ user.OPTS,
+ network.OPTS,
+ instance.OPTS,
+ alarm.OPTS))
]
-def prepare_conf(conf=None):
+def prepare_conf(args=None, conf=None, config_files=None):
if conf is None:
conf = cfg.ConfigOpts()
@@ -27,4 +46,7 @@ def prepare_conf(conf=None):
conf.register_opts(list(options),
group=None if group == 'DEFAULT' else group)
+ conf(args, project='doctor', validate_default_values=True,
+ default_config_files=config_files)
+
return conf
diff --git a/tests/consumer/__init__.py b/tests/consumer/__init__.py
new file mode 100644
index 00000000..ccec8644
--- /dev/null
+++ b/tests/consumer/__init__.py
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+ cfg.StrOpt('type',
+ default='sample',
+ choices=['sample'],
+ help='the component of doctor consumer',
+ required=True),
+ cfg.StrOpt('ip',
+ default='127.0.0.1',
+ help='the ip of consumer',
+ required=True),
+ cfg.IntOpt('port',
+ default='12346',
+ help='the port of doctor consumer',
+ required=True),
+]
+
+
+_consumer_name_class_mapping = {
+ 'sample': 'consumer.sample.SampleConsumer'
+}
+
+
+def get_consumer(conf, log):
+ consumer_class = _consumer_name_class_mapping.get(conf.consumer.type)
+ return importutils.import_object(consumer_class, conf, log) \ No newline at end of file
diff --git a/tests/consumer/base.py b/tests/consumer/base.py
new file mode 100644
index 00000000..35170748
--- /dev/null
+++ b/tests/consumer/base.py
@@ -0,0 +1,26 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseConsumer(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass \ No newline at end of file
diff --git a/tests/consumer/sample.py b/tests/consumer/sample.py
new file mode 100644
index 00000000..a698623a
--- /dev/null
+++ b/tests/consumer/sample.py
@@ -0,0 +1,71 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from consumer.base import BaseConsumer
+
+
+class SampleConsumer(BaseConsumer):
+
+ def __init__(self, conf, log):
+ super(SampleConsumer, self).__init__(conf, log)
+ self.app = None
+
+ def start(self):
+ self.log.info('sample consumer start......')
+ self.app = ConsumerApp(self.conf.consumer.port, self, self.log)
+ self.app.start()
+
+ def stop(self):
+ self.log.info('sample consumer stop......')
+ if not self.app:
+ return
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ }
+ url = 'http://%s:%d/shutdown'\
+ % (self.conf.consumer.ip,
+ self.conf.consumer.port)
+ requests.post(url, data='', headers=headers)
+
+
+class ConsumerApp(Thread):
+
+ def __init__(self, port, consumer, log):
+ Thread.__init__(self)
+ self.port = port
+ self.consumer = consumer
+ self.log = log
+
+ def run(self):
+ app = Flask('consumer')
+
+ @app.route('/failure', methods=['POST'])
+ def event_posted():
+ self.log.info('doctor consumer notified at %s' % time.time())
+ self.log.info('received data = %s' % request.data)
+ data = json.loads(request.data)
+ return "OK"
+
+ @app.route('/shutdown', methods=['POST'])
+ def shutdown():
+ self.log.info('shutdown consumer app server at %s' % time.time())
+ func = request.environ.get('werkzeug.server.shutdown')
+ if func is None:
+ raise RuntimeError('Not running with the Werkzeug Server')
+ func()
+ return 'consumer app shutting down...'
+
+ app.run(host="0.0.0.0", port=self.port) \ No newline at end of file
diff --git a/tests/identity_auth.py b/tests/identity_auth.py
index ffecc68a..c94893f4 100644
--- a/tests/identity_auth.py
+++ b/tests/identity_auth.py
@@ -13,13 +13,14 @@ from keystoneauth1 import loading
from keystoneauth1 import session
-def get_identity_auth():
+def get_identity_auth(username=None, password=None, project=None):
auth_url = os.environ['OS_AUTH_URL']
- username = os.environ['OS_USERNAME']
- password = os.environ['OS_PASSWORD']
+ username = username or os.environ['OS_USERNAME']
+ password = password or os.environ['OS_PASSWORD']
user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default'
user_domain_id = os.environ.get('OS_USER_DOMAIN_ID') or 'default'
- project_name = os.environ.get('OS_PROJECT_NAME') or os.environ.get('OS_TENANT_NAME')
+ project_name = project or os.environ.get('OS_PROJECT_NAME') \
+ or os.environ.get('OS_TENANT_NAME')
project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default'
project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID') or 'default'
diff --git a/tests/image.py b/tests/image.py
index 0b4a3d72..453322b8 100644
--- a/tests/image.py
+++ b/tests/image.py
@@ -7,71 +7,68 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
import os
-import urllib2
+import urllib.request
from oslo_config import cfg
from identity_auth import get_session
from os_clients import glance_client
-import logger as doctor_log
-IMAGE_OPTS = [
- cfg.StrOpt('name',
+OPTS = [
+ cfg.StrOpt('image_name',
default=os.environ.get('IMAGE_NAME', 'cirros'),
help='the name of test image',
required=True),
- cfg.StrOpt('format',
+ cfg.StrOpt('image_format',
default='qcow2',
help='the format of test image',
required=True),
- cfg.StrOpt('file_name',
+ cfg.StrOpt('image_filename',
default='cirros.img',
help='the name of image file',
required=True),
- cfg.StrOpt('url',
+ cfg.StrOpt('image_download_url',
default='https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img',
help='the url where to get the image',
required=True),
]
-LOG = doctor_log.Logger('doctor').getLogger()
-
class Image(object):
- def __init__(self, conf):
+ def __init__(self, conf, log):
self.conf = conf
+ self.log = log
self.glance = \
- glance_client(conf.os_clients.glance_version,
- get_session())
+ glance_client(conf.glance_version, get_session())
self.use_existing_image = False
self.image = None
def create(self):
- LOG.info('image create start......')
+ self.log.info('image create start......')
images = {image.name: image for image in self.glance.images.list()}
- if self.conf.image.name not in images:
- if not os.path.exists(self.conf.image.file_name):
- resp = urllib2.urlopen(self.conf.image.url)
- with open(self.conf.image.file_name, "wb") as file:
+ if self.conf.image_name not in images:
+ if not os.path.exists(self.conf.image_filename):
+ resp = urllib.request.urlopen(self.conf.image_download_url)
+ with open(self.conf.image_filename, "wb") as file:
file.write(resp.read())
- self.image = self.glance.images.create(name=self.conf.image.name,
- disk_format=self.conf.image.format,
+ self.image = self.glance.images.create(name=self.conf.image_name,
+ disk_format=self.conf.image_format,
container_format="bare",
visibility="public")
self.glance.images.upload(self.image['id'],
- open(self.conf.image.file_name, 'rb'))
+ open(self.conf.image_filename, 'rb'))
else:
self.use_existing_image = True
- self.image = images[self.conf.image.name]
+ self.image = images[self.conf.image_name]
- LOG.info('image create end......')
+ self.log.info('image create end......')
def delete(self):
- LOG.info('image delete start.......')
+ self.log.info('image delete start.......')
if not self.use_existing_image and self.image:
self.glance.images.delete(self.image['id'])
- LOG.info('image delete end.......')
+ self.log.info('image delete end.......')
diff --git a/tests/inspector.py b/tests/inspector.py
index d11da299..82ffc338 100644
--- a/tests/inspector.py
+++ b/tests/inspector.py
@@ -54,8 +54,7 @@ class DoctorInspectorSample(object):
# Pool of novaclients for redundant usage
for i in range(self.NUMBER_OF_CLIENTS):
self.novaclients.append(
- novaclient.Client(self.NOVA_API_VERSION, session=sess,
- connection_pool=True))
+ novaclient.Client(self.NOVA_API_VERSION, session=sess))
# Normally we use this client for non redundant API calls
self.nova=self.novaclients[0]
self.nova.servers.list(detailed=False)
@@ -117,8 +116,7 @@ def get_args():
def main():
args = get_args()
- app.run(port=args.port)
-
+ app.run(host='0.0.0.0', port=args.port)
if __name__ == '__main__':
main()
diff --git a/tests/inspector/__init__.py b/tests/inspector/__init__.py
new file mode 100644
index 00000000..afba4800
--- /dev/null
+++ b/tests/inspector/__init__.py
@@ -0,0 +1,40 @@
+#############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+ cfg.StrOpt('type',
+ default=os.environ.get('INSPECTOR_TYPE', 'sample'),
+ choices=['sample', 'congress', 'vitrage'],
+ help='the component of doctor inspector',
+ required=True),
+ cfg.StrOpt('ip',
+ default='127.0.0.1',
+ help='the host ip of inspector',
+ required=False),
+ cfg.StrOpt('port',
+ default='12345',
+ help='the port of default for inspector',
+ required=False),
+]
+
+
+_inspector_name_class_mapping = {
+ 'sample': 'inspector.sample.SampleInspector',
+ 'congress': 'inspector.congress.CongressInspector',
+}
+
+
+def get_inspector(conf, log):
+ inspector_class = _inspector_name_class_mapping[conf.inspector.type]
+ return importutils.import_object(inspector_class, conf, log)
diff --git a/tests/inspector/base.py b/tests/inspector/base.py
new file mode 100644
index 00000000..854f0695
--- /dev/null
+++ b/tests/inspector/base.py
@@ -0,0 +1,30 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInspector(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractmethod
+ def get_inspector_url(self):
+ pass
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass \ No newline at end of file
diff --git a/tests/inspector/congress.py b/tests/inspector/congress.py
new file mode 100644
index 00000000..ae295852
--- /dev/null
+++ b/tests/inspector/congress.py
@@ -0,0 +1,94 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import congress_client
+
+from inspector.base import BaseInspector
+
+
+class CongressInspector(BaseInspector):
+ nova_api_min_version = '2.11'
+ doctor_driver = 'doctor'
+ doctor_datasource = 'doctor'
+ policy = 'classification'
+ rules = {
+ 'host_down':
+ 'host_down(host) :- doctor:events(hostname=host, type="compute.host.down", status="down")',
+ 'active_instance_in_host':
+ 'active_instance_in_host(vmid, host) :- nova:servers(id=vmid, host_name=host, status="ACTIVE")',
+ 'host_force_down':
+ 'execute[nova:services.force_down(host, "nova-compute", "True")] :- host_down(host)',
+ 'error_vm_states':
+ 'execute[nova:servers.reset_state(vmid, "error")] :- host_down(host), active_instance_in_host(vmid, host)'
+ }
+
+ def __init__(self, conf, log):
+ super(CongressInspector, self).__init__(conf, log)
+ self.auth = get_identity_auth()
+ self.congress = congress_client(get_session(auth=self.auth))
+ self._init_driver_and_ds()
+ self.inspector_url = self.get_inspector_url()
+
+ def _init_driver_and_ds(self):
+ datasources = \
+ {ds['name']: ds for ds in self.congress.list_datasources()['results']}
+
+ # check nova_api version
+ nova_api_version = datasources['nova']['config'].get('api_version')
+ if nova_api_version and nova_api_version < self.nova_api_min_version:
+ raise Exception('Congress Nova datasource API version < nova_api_min_version(%s)'
+ % self.nova_api_min_version)
+
+ # create doctor datasource if it's not exist
+ if self.doctor_datasource not in datasources:
+ self.congress.create_datasource(
+ body={'driver': self.doctor_driver,
+ 'name': self.doctor_datasource})
+
+ # check whether doctor driver exist
+ drivers = \
+ {driver['id']: driver for driver in self.congress.list_drivers()['results']}
+ if self.doctor_driver not in drivers:
+ raise Exception('Do not support doctor driver in congress')
+
+ self.policy_rules = \
+ {rule['name']: rule for rule in
+ self.congress.list_policy_rules(self.policy)['results']}
+
+ def get_inspector_url(self):
+ ds = self.congress.list_datasources()['results']
+ doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+ None)
+ congress_endpoint = self.congress.httpclient.get_endpoint(auth=self.auth)
+ return ('%s/v1/data-sources/%s/tables/events/rows' %
+ (congress_endpoint, doctor_ds['id']))
+
+ def start(self):
+ self.log.info('congress inspector start......')
+
+ for rule_name, rule in self.rules.items():
+ self._add_rule(rule_name, rule)
+
+ def stop(self):
+ self.log.info('congress inspector stop......')
+
+ for rule_name in self.rules.keys():
+ self._del_rule(rule_name)
+
+ def _add_rule(self, rule_name, rule):
+ if rule_name not in self.policy_rules:
+ self.congress.create_policy_rule(self.policy,
+ body={'name': rule_name,
+ 'rule': rule})
+
+ def _del_rule(self, rule_name):
+ if rule_name in self.policy_rules:
+ rule_id = self.policy_rules[rule_name]['id']
+ self.congress.delete_policy_rule(self.policy, rule_id)
diff --git a/tests/inspector/sample.py b/tests/inspector/sample.py
new file mode 100644
index 00000000..dda053ab
--- /dev/null
+++ b/tests/inspector/sample.py
@@ -0,0 +1,151 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import collections
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import nova_client
+from inspector.base import BaseInspector
+
+
+class SampleInspector(BaseInspector):
+ event_type = 'compute.host.down'
+
+ def __init__(self, conf, log):
+ super(SampleInspector, self).__init__(conf, log)
+ self.inspector_url = self.get_inspector_url()
+ self.novaclients = list()
+ self._init_novaclients()
+ # Normally we use this client for non redundant API calls
+ self.nova = self.novaclients[0]
+
+ self.servers = collections.defaultdict(list)
+ self.hostnames = list()
+ self.app = None
+
+ def _init_novaclients(self):
+ self.NUMBER_OF_CLIENTS = self.conf.instance_count
+ auth = get_identity_auth(project=self.conf.doctor_project)
+ session = get_session(auth=auth)
+ for i in range(self.NUMBER_OF_CLIENTS):
+ self.novaclients.append(
+ nova_client(self.conf.nova_version, session))
+
+ def _init_servers_list(self):
+ self.servers.clear()
+ opts = {'all_tenants': True}
+ servers = self.nova.servers.list(search_opts=opts)
+ for server in servers:
+ try:
+ host = server.__dict__.get('OS-EXT-SRV-ATTR:host')
+ self.servers[host].append(server)
+ self.log.debug('get hostname=%s from server=%s' % (host, server))
+ except Exception as e:
+ self.log.info('can not get hostname from server=%s' % server)
+
+ def get_inspector_url(self):
+ return 'http://%s:%s' % (self.conf.inspector.ip, self.conf.inspector.port)
+
+ def start(self):
+ self.log.info('sample inspector start......')
+ self._init_servers_list()
+ self.app = InspectorApp(self.conf.inspector.port, self, self.log)
+ self.app.start()
+
+ def stop(self):
+ self.log.info('sample inspector stop......')
+ if not self.app:
+ return
+ for hostname in self.hostnames:
+ self.nova.services.force_down(hostname, 'nova-compute', False)
+
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ }
+ url = '%s%s' % (self.inspector_url, 'shutdown') \
+ if self.inspector_url.endswith('/') else \
+ '%s%s' % (self.inspector_url, '/shutdown')
+ requests.post(url, data='', headers=headers)
+
+ def handle_events(self, events):
+ for event in events:
+ hostname = event['details']['hostname']
+ event_type = event['type']
+ if event_type == self.event_type:
+ self.hostnames.append(hostname)
+ self.disable_compute_host(hostname)
+
+ def disable_compute_host(self, hostname):
+ threads = []
+ if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS:
+ # TODO(tojuvone): This could be enhanced in future with dynamic
+ # reuse of self.novaclients when all threads in use
+ self.log.error('%d servers in %s. Can handle only %d'%(
+ self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS))
+ for nova, server in zip(self.novaclients, self.servers[hostname]):
+ t = ThreadedResetState(nova, "error", server, self.log)
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+ self.nova.services.force_down(hostname, 'nova-compute', True)
+ self.log.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
+
+
+class ThreadedResetState(Thread):
+
+ def __init__(self, nova, state, server, log):
+ Thread.__init__(self)
+ self.nova = nova
+ self.state = state
+ self.server = server
+ self.log = log
+
+ def run(self):
+ self.nova.servers.reset_state(self.server, self.state)
+ self.log.info('doctor mark vm(%s) error at %s' % (self.server, time.time()))
+
+
+class InspectorApp(Thread):
+
+ def __init__(self, port, inspector, log):
+ Thread.__init__(self)
+ self.port = port
+ self.inspector = inspector
+ self.log = log
+
+ def run(self):
+ app = Flask('inspector')
+
+ @app.route('/events', methods=['PUT'])
+ def event_posted():
+ self.log.info('event posted in sample inspector at %s' % time.time())
+ self.log.info('sample inspector = %s' % self.inspector)
+ self.log.info('sample inspector received data = %s' % request.data)
+ events = json.loads(request.data.decode('utf8'))
+ self.inspector.handle_events(events)
+ return "OK"
+
+ @app.route('/shutdown', methods=['POST'])
+ def shutdown():
+ self.log.info('shutdown inspector app server at %s' % time.time())
+ func = request.environ.get('werkzeug.server.shutdown')
+ if func is None:
+ raise RuntimeError('Not running with the Werkzeug Server')
+ func()
+ return 'inspector app shutting down...'
+
+ app.run(host="0.0.0.0", port=self.port)
diff --git a/tests/installer/__init__.py b/tests/installer/__init__.py
new file mode 100644
index 00000000..bb0e452d
--- /dev/null
+++ b/tests/installer/__init__.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+ cfg.StrOpt('type',
+ default=os.environ.get('INSTALLER_TYPE', 'local'),
+ choices=['local', 'apex'],
+ help='the type of installer',
+ required=True),
+ cfg.StrOpt('ip',
+ default=os.environ.get('INSTALLER_IP', '127.0.0.1'),
+ help='the ip of installer'),
+ cfg.StrOpt('username',
+ default='root',
+ help='the user name for login installer server',
+ required=True),
+]
+
+
+_installer_name_class_mapping = {
+ 'local': 'installer.local.LocalInstaller',
+ 'apex': 'installer.apex.ApexInstaller'
+}
+
+
+def get_installer(conf, log):
+ installer_class = _installer_name_class_mapping[conf.installer.type]
+ return importutils.import_object(installer_class, conf, log) \ No newline at end of file
diff --git a/tests/installer/apex.py b/tests/installer/apex.py
new file mode 100644
index 00000000..24cd5a75
--- /dev/null
+++ b/tests/installer/apex.py
@@ -0,0 +1,99 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import getpass
+import grp
+import os
+import pwd
+import stat
+import sys
+
+from installer.common.congress import set_doctor_driver_conf
+from installer.common.congress import restore_doctor_driver_conf
+from installer.base import BaseInstaller
+from utils import SSHClient
+
+
+class ApexInstaller(BaseInstaller):
+ node_user_name = 'heat-admin'
+ cm_set_script = 'set_ceilometer.py'
+ cm_restore_script = 'restore_ceilometer.py'
+
+ def __init__(self, conf, log):
+ super(ApexInstaller, self).__init__(conf, log)
+ self.client = SSHClient(self.conf.installer.ip,
+ self.conf.installer.username,
+ look_for_keys=True)
+ self.key_file = None
+ self.controllers = list()
+ self.controller_clients = list()
+
+ def setup(self):
+ self.log.info('Setup Apex installer start......')
+
+ self.key_file = self.get_ssh_key_from_installer()
+ self.get_controller_ips()
+ self.set_apply_patches()
+
+ def cleanup(self):
+ self.restore_apply_patches()
+
+ def get_ssh_key_from_installer(self):
+ self.log.info('Get SSH keys from Apex installer......')
+
+ self.client.scp('/home/stack/.ssh/id_rsa', './instack_key', method='get')
+ user = getpass.getuser()
+ uid = pwd.getpwnam(user).pw_uid
+ gid = grp.getgrnam(user).gr_gid
+ os.chown('./instack_key', uid, gid)
+ os.chmod('./instack_key', stat.S_IREAD)
+ current_dir = sys.path[0]
+ return '{0}/{1}'.format(current_dir, 'instack_key')
+
+ def get_controller_ips(self):
+ self.log.info('Get controller ips from Apex installer......')
+
+ command = "source stackrc; " \
+ "nova list | grep ' overcloud-controller-[0-9] ' " \
+ "| sed -e 's/^.*ctlplane=//' |awk '{print $1}'"
+ ret, controllers = self.client.ssh(command)
+ if ret:
+ raise Exception('Exec command to get controller ips in Apex installer failed'
+ 'ret=%s, output=%s' % (ret, controllers))
+ self.controllers = controllers
+
+ def set_apply_patches(self):
+ self.log.info('Set apply patches start......')
+
+ for node_ip in self.controllers:
+ client = SSHClient(node_ip, self.node_user_name, key_filename=self.key_file)
+ self.controller_clients.append(client)
+ self._ceilometer_apply_patches(client, self.cm_set_script)
+ cmd = 'sudo systemctl restart openstack-congress-server.service'
+ set_doctor_driver_conf(client, cmd)
+
+ def restore_apply_patches(self):
+ self.log.info('restore apply patches start......')
+
+ for client in self.controller_clients:
+ self._ceilometer_apply_patches(client, self.cm_restore_script)
+ cmd = 'sudo systemctl restart openstack-congress-server.service'
+ restore_doctor_driver_conf(client, cmd)
+
+ def _ceilometer_apply_patches(self, ssh_client, script_name):
+ installer_dir = os.path.dirname(os.path.realpath(__file__))
+ script_abs_path = '{0}/{1}/{2}'.format(installer_dir, 'common', script_name)
+
+ ssh_client.scp(script_abs_path, script_name)
+ cmd = 'sudo python %s' % script_name
+ ret, output = ssh_client.ssh(cmd)
+ if ret:
+ raise Exception('Do the ceilometer command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cmd, output))
+ ssh_client.ssh('sudo systemctl restart openstack-ceilometer-notification.service')
+
diff --git a/tests/installer/base.py b/tests/installer/base.py
new file mode 100644
index 00000000..f3837f15
--- /dev/null
+++ b/tests/installer/base.py
@@ -0,0 +1,32 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInstaller(object):
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractproperty
+ def node_user_name(self):
+ """user name for login to cloud node"""
+
+ @abc.abstractmethod
+ def get_ssh_key_from_installer(self):
+ pass
+
+ @abc.abstractmethod
+ def setup(self):
+ pass
+
+ @abc.abstractmethod
+ def cleanup(self):
+ pass
diff --git a/tests/installer/common/congress.py b/tests/installer/common/congress.py
new file mode 100644
index 00000000..db882de2
--- /dev/null
+++ b/tests/installer/common/congress.py
@@ -0,0 +1,47 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+def set_doctor_driver_conf(ssh_client, restart_cmd):
+ cg_set_cmd = '''#!/bin/bash
+co_conf=/etc/congress/congress.conf
+co_conf_bak=/etc/congress/congress.conf.bak
+co_entry="congress.datasources.doctor_driver.DoctorDriver"
+if sudo grep -e "^drivers.*$co_entry" $co_conf; then
+ echo "NOTE: congress is configured as we needed"
+else
+ echo "modify the congress config"
+ sudo cp $co_conf $co_conf_bak
+ sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf
+ %s
+fi
+ ''' % (restart_cmd)
+
+ ret, output = ssh_client.ssh(cg_set_cmd)
+ if ret:
+ raise Exception('Do the congress command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cg_set_cmd, output))
+
+
+def restore_doctor_driver_conf(ssh_client, restart_cmd):
+ cg_restore_cmd = '''#!/bin/bash
+co_conf=/etc/congress/congress.conf
+co_conf_bak=/etc/congress/congress.conf.bak
+if [ -e $co_conf_bak ]; then
+ echo "restore the congress config"
+ sudo cp $co_conf_bak $co_conf
+ sudo rm $co_conf_bak
+ %s
+else
+ echo "Do not need to restore the congress config"
+fi
+ ''' % (restart_cmd)
+
+ ret, output = ssh_client.ssh(cg_restore_cmd)
+ if ret:
+ raise Exception('Do the congress command in controller node failed....'
+ 'ret=%s, cmd=%s, output=%s' % (ret, cg_restore_cmd, output))
diff --git a/tests/installer/common/restore_ceilometer.py b/tests/installer/common/restore_ceilometer.py
new file mode 100644
index 00000000..d25b9ede
--- /dev/null
+++ b/tests/installer/common/restore_ceilometer.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+
+
+def restore_ep_config():
+
+ if not os.path.isfile(ep_file_bak):
+ print('Bak_file:%s does not exist.' % ep_file_bak)
+ else:
+ print('restore')
+ shutil.copyfile(ep_file_bak, ep_file)
+ os.remove(ep_file_bak)
+ return
+
+
+restore_ep_config()
diff --git a/tests/installer/common/set_ceilometer.py b/tests/installer/common/set_ceilometer.py
new file mode 100644
index 00000000..f5946cb2
--- /dev/null
+++ b/tests/installer/common/set_ceilometer.py
@@ -0,0 +1,44 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+import yaml
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+event_notifier_topic = 'notifier://?topic=alarm.all'
+
+
+def set_notifier_topic():
+ config_modified = False
+
+ if not os.path.isfile(ep_file):
+ raise Exception("File doesn't exist: %s." % ep_file)
+
+ with open(ep_file, 'r') as file:
+ config = yaml.safe_load(file)
+
+ sinks = config['sinks']
+ for sink in sinks:
+ if sink['name'] == 'event_sink':
+ publishers = sink['publishers']
+ if event_notifier_topic not in publishers:
+ print('Add event notifier in ceilometer')
+ publishers.append(event_notifier_topic)
+ config_modified = True
+ else:
+ print('NOTE: event notifier is configured in ceilometer as we needed')
+
+ if config_modified:
+ shutil.copyfile(ep_file, ep_file_bak)
+ with open(ep_file, 'w+') as file:
+ file.write(yaml.safe_dump(config))
+
+
+set_notifier_topic()
diff --git a/tests/installer/local.py b/tests/installer/local.py
new file mode 100644
index 00000000..abe0ba25
--- /dev/null
+++ b/tests/installer/local.py
@@ -0,0 +1,97 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+
+from installer.base import BaseInstaller
+from utils import load_json_file
+from utils import write_json_file
+
+
+class LocalInstaller(BaseInstaller):
+ node_user_name = 'root'
+
+ nova_policy_file = '/etc/nova/policy.json'
+ nova_policy_file_backup = '%s%s' % (nova_policy_file, '.bak')
+
+ def __init__(self, conf, log):
+ super(LocalInstaller, self).__init__(conf, log)
+ self.policy_modified = False
+ self.add_policy_file = False
+
+ def setup(self):
+ self.get_ssh_key_from_installer()
+ self.set_apply_patches()
+
+ def cleanup(self):
+ self.restore_apply_patches()
+
+ def get_ssh_key_from_installer(self):
+ self.log.info('Assuming SSH keys already exchanged with computer for local installer type')
+ return
+
+ def set_apply_patches(self):
+ self._set_nova_policy()
+
+ def restore_apply_patches(self):
+ self._restore_nova_policy()
+
+ def _set_nova_policy(self):
+ host_status_policy = 'os_compute_api:servers:show:host_status'
+ host_status_rule = 'rule:admin_or_owner'
+ policy_data = {
+ 'context_is_admin': 'role:admin',
+ 'owner': 'user_id:%(user_id)s',
+ 'admin_or_owner': 'rule:context_is_admin or rule:owner',
+ host_status_policy: host_status_rule
+ }
+
+ if os.path.isfile(self.nova_policy_file):
+ data = load_json_file(self.nova_policy_file)
+ if host_status_policy in data:
+ rule_origion = data[host_status_policy]
+ if host_status_rule == rule_origion:
+ self.log.info('Do not need to modify nova policy.')
+ self.policy_modified = False
+ else:
+ # update the host_status_policy
+ data[host_status_policy] = host_status_rule
+ self.policy_modified = True
+ else:
+ # add the host_status_policy, if the admin_or_owner is not
+ # defined, add it also
+ for policy, rule in policy_data.items():
+ if policy not in data:
+ data[policy] = rule
+ self.policy_modified = True
+ if self.policy_modified:
+ self.log.info('Nova policy is Modified.')
+ shutil.copyfile(self.nova_policy_file,
+ self.nova_policy_file_backup)
+ else:
+ # file does not exit, create a new one and add the policy
+ self.log.info('Nova policy file not exist. Creating a new one')
+ data = policy_data
+ self.add_policy_file = True
+
+ if self.policy_modified or self.add_policy_file:
+ write_json_file(self.nova_policy_file, data)
+ os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"')
+
+ def _restore_nova_policy(self):
+ if self.policy_modified:
+ shutil.copyfile(self.nova_policy_file_backup, self.nova_policy_file)
+ os.remove(self.nova_policy_file_backup)
+ elif self.add_policy_file:
+ os.remove(self.nova_policy_file)
+
+ if self.add_policy_file or self.policy_modified:
+ os.system('screen -S stack -p n-api -X stuff "^C^M^[[A^M"')
+ self.add_policy_file = False
+ self.policy_modified = False
diff --git a/tests/instance.py b/tests/instance.py
new file mode 100644
index 00000000..c6acbc3d
--- /dev/null
+++ b/tests/instance.py
@@ -0,0 +1,114 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import time
+
+from oslo_config import cfg
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import neutron_client
+from os_clients import nova_client
+
+OPTS = [
+ cfg.StrOpt('flavor',
+ default='m1.tiny',
+ help='the name of flavor',
+ required=True),
+ cfg.IntOpt('instance_count',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the count of instance',
+ required=True),
+ cfg.StrOpt('instance_basename',
+ default='doctor_vm',
+ help='the base name of instance',
+ required=True),
+]
+
+
+class Instance(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(username=self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ project=self.conf.doctor_project)
+ self.nova = \
+ nova_client(conf.nova_version,
+ get_session(auth=self.auth))
+ self.neutron = neutron_client(get_session(auth=self.auth))
+ self.servers = {}
+ self.vm_names = []
+
+ def create(self):
+ self.log.info('instance create start......')
+
+ # get flavor, image and network for vm boot
+ flavors = {flavor.name: flavor for flavor in self.nova.flavors.list()}
+ flavor = flavors.get(self.conf.flavor)
+ image = self.nova.glance.find_image(self.conf.image_name)
+ network = self.neutron.list_networks(name=self.conf.net_name)['networks'][0]
+ nics = {'net-id': network['id']}
+
+ self.servers = \
+ {getattr(server, 'name'): server
+ for server in self.nova.servers.list()}
+ for i in range(0, self.conf.instance_count):
+ vm_name = "%s%d"%(self.conf.instance_basename, i)
+ self.vm_names.append(vm_name)
+ if vm_name not in self.servers:
+ server = self.nova.servers.create(vm_name, image,
+ flavor, nics=[nics])
+ self.servers[vm_name] = server
+ time.sleep(0.1)
+
+ self.log.info('instance create end......')
+
+ def delete(self):
+ self.log.info('instance delete start.......')
+
+ for vm_name in self.vm_names:
+ if vm_name in self.servers:
+ self.nova.servers.delete(self.servers[vm_name])
+ time.sleep(0.1)
+
+ # check that all vms are deleted
+ while self.nova.servers.list():
+ time.sleep(0.1)
+ self.servers.clear()
+ del self.vm_names[:]
+
+ self.log.info('instance delete end.......')
+
+ def wait_for_vm_launch(self):
+ self.log.info('wait for vm launch start......')
+
+ wait_time = 60
+ count = 0
+ while count < wait_time:
+ active_count = 0
+ for vm_name in self.vm_names:
+ server = self.nova.servers.get(self.servers[vm_name])
+ server_status = getattr(server, 'status').lower()
+ if 'active' == server_status:
+ active_count += 1
+ elif 'error' == server_status:
+ raise Exception('vm launched with error state')
+ else:
+ time.sleep(2)
+ count += 1
+ continue
+ if active_count == self.conf.instance_count:
+ self.log.info('wait for vm launch end......')
+ return
+ count += 1
+ time.sleep(2)
+ raise Exception('time out for vm launch')
+
diff --git a/tests/lib/installers/apex b/tests/lib/installers/apex
index 55878dec..f7b9624e 100644
--- a/tests/lib/installers/apex
+++ b/tests/lib/installers/apex
@@ -5,7 +5,7 @@ ssh_opts_cpu="$ssh_opts -i instack_key"
function get_installer_ip {
is_set INSTALLER_IP && return
- INSTALLER_IP=$(get_first_vnic_ip instack)
+ INSTALLER_IP=$(get_first_vnic_ip undercloud)
}
function installer_get_ssh_keys {
@@ -44,13 +44,14 @@ function installer_apply_patches {
fi
co_conf=/etc/congress/congress.conf
+ co_conf_bak=/etc/congress/congress.conf.bak
co_entry="congress.datasources.doctor_driver.DoctorDriver"
if sudo grep -e "^drivers.*$co_entry" $co_conf; then
echo "NOTE: congress is configured as we needed"
else
echo "modify the congress config"
- sudo sed -i -e "/^drivers/s/$/,$co_entry # added by doctor script/" \
- $co_conf
+ sudo cp $co_conf $co_conf_bak
+ sudo sed -i -e "/^drivers/s/$/,$co_entry/" $co_conf
sudo systemctl restart openstack-congress-server.service
fi
' > installer_apply_patches_$node.log 2>&1
@@ -90,10 +91,10 @@ function installer_revert_patches {
date
co_conf=/etc/congress/congress.conf
- co_entry="congress.datasources.doctor_driver.DoctorDriver"
- if sudo grep -q -e "# added by doctor script" $co_conf; then
- echo "modify the congress config"
- sudo sed -i -e "/^drivers/s/^\(.*\),$co_entry # added by doctor script/\1/" $co_conf
+ co_conf_bak=/etc/congress/congress.conf.bak
+ if [ -e $co_conf_bak ]; then
+ echo "restore the congress config"
+ sudo mv $co_conf_bak $co_conf
sudo systemctl restart openstack-congress-server.service
fi
diff --git a/tests/lib/installers/fuel b/tests/lib/installers/fuel
index 0c56963c..85865720 100644
--- a/tests/lib/installers/fuel
+++ b/tests/lib/installers/fuel
@@ -96,8 +96,15 @@ function installer_apply_patches {
service nova-api restart
fi
else
- # TODO(tojuvone) policy.json might not exists in Ocata.
- echo "$np_conf does not exist!!!"
+ # policy.json does not exist in Ocata.
+ echo "$np_conf does not exist. Creating new one."
+ echo -e "{\n \"context_is_admin\": \"role:admin\"," > $np_conf
+ echo -e " \"owner\" : \"user_id:%(user_id)s\"," >> $np_conf
+ echo -e " \"admin_or_owner\": \"rule:context_is_admin or rule:owner\"," >> $np_conf
+ echo -e " \"os_compute_api:servers:show:host_status\": \"rule:admin_or_owner\" \n}" >> $np_conf
+ np_rm="${np_conf}-doctor-rm"
+ cp $np_conf $np_rm
+ service nova-api restart
fi
' > installer_apply_patches_$node.log 2>&1
done
@@ -167,14 +174,16 @@ function installer_revert_patches {
fi
np_conf=/etc/nova/policy.json
- entry="os_compute_api:servers:show:host_status"
- if [ -e $np_conf ]; then
- np_backup="${np_conf}-doctor-saved"
- if [ -e $np_backup ]; then
- cp -f $np_backup $np_conf
- rm $np_backup
- service nova-api restart
- fi
+ np_backup="${np_conf}-doctor-saved"
+ np_rm="${np_conf}-doctor-rm"
+ if [ -e $np_backup ]; then
+ cp -f $np_backup $np_conf
+ rm $np_backup
+ service nova-api restart
+ elif [ -e $np_rm ]; then
+ rm $np_conf
+ rm $np_rm
+ service nova-api restart
fi
' >> installer_apply_patches_$node.log 2>&1
done
diff --git a/tests/lib/installers/local b/tests/lib/installers/local
index 50c3686f..d628867a 100644
--- a/tests/lib/installers/local
+++ b/tests/lib/installers/local
@@ -9,7 +9,42 @@ function installer_get_ssh_keys {
}
function installer_apply_patches {
- # Noop
+ set -x
+ date
+ echo "### apply patches (installer=local)"
+ np_conf=/etc/nova/policy.json
+ if [ -e $np_conf ]; then
+ entry="os_compute_api:servers:show:host_status"
+ new="rule:admin_or_owner"
+ np_backup="${np_conf}-doctor-saved"
+ if grep -q "${entry}.*${new}" $np_conf; then
+ echo "Not modifying nova policy"
+ elif grep -q "${entry}" $np_conf; then
+ echo "modify nova policy"
+ cp $np_conf $np_backup
+ oldline=$(grep "$entry" $np_conf)
+ newline=$(echo "$oldline" | sed "s/rule.*\"/$new\"/")
+ sed -i "s/$oldline/$newline/" $np_conf
+ # TODO(umar): Update to systemd when screen is no more used for devstack
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M" # restart n-api service
+ else
+ echo "add nova policy"
+ cp $np_conf $np_backup
+ sed -i "/{/a \ \"${entry}\": \"$new\"" $np_conf
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M"
+ fi
+ else
+ # policy.json does not exist in Ocata.
+ echo "$np_conf does not exist. Creating a new one"
+ echo -e '{\n "context_is_admin": "role:admin",' > $np_conf
+ echo -e ' "owner" : "user_id:%(user_id)s",' >> $np_conf
+ echo -e ' "admin_or_owner": "rule:context_is_admin or rule:owner",' >> $np_conf
+ echo -e ' "os_compute_api:servers:show:host_status": "rule:admin_or_owner"\n}' >> $np_conf
+ np_rm="${np_conf}-doctor-rm"
+ cp $np_conf $np_rm
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M"
+ fi
+
return
}
@@ -31,6 +66,22 @@ function get_compute_ip_from_hostname {
}
function cleanup_installer {
- # Noop
+ set -x
+ echo "### revert patches (installer=local)"
+ date
+
+ np_conf=/etc/nova/policy.json
+ np_backup="${np_conf}-doctor-saved"
+ np_rm="${np_conf}-doctor-rm"
+ if [ -e $np_backup ]; then
+ cp -f $np_backup $np_conf
+ rm $np_backup
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M"
+ elif [ -e $np_rm ]; then
+ rm $np_conf
+ rm $np_rm
+ screen -S stack -p n-api -X stuff "^C^M^[[A^M"
+ fi
+
return
}
diff --git a/tests/lib/monitor b/tests/lib/monitor
new file mode 100644
index 00000000..6b804ec2
--- /dev/null
+++ b/tests/lib/monitor
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+MONITOR_TYPE=${MONITOR_TYPE:-sample}
+
+function is_monitor_supported {
+ local monitor="$1"
+ [[ -f $TOP_DIR/lib/monitors/$monitor/$monitor ]]
+}
+
+function is_monitor {
+ local monitor="$1"
+ [[ $monitor == $MONITOR_TYPE ]]
+}
+
+function start_monitor {
+ start_monitor_$MONITOR_TYPE
+}
+
+function stop_monitor {
+ stop_monitor_$MONITOR_TYPE
+}
+
+function cleanup_monitor {
+ cleanup_monitor_$MONITOR_TYPE
+}
+
+if ! is_monitor_supported $MONITOR_TYPE; then
+ die $LINENO "MONITOR_TYPE=$MONITOR_TYPE is not supported."
+fi
+
+source $TOP_DIR/lib/monitors/$MONITOR_TYPE/$MONITOR_TYPE
diff --git a/tests/lib/monitors/collectd/collectd b/tests/lib/monitors/collectd/collectd
new file mode 100644
index 00000000..f5096658
--- /dev/null
+++ b/tests/lib/monitors/collectd/collectd
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+function start_monitor_collectd {
+ ## CONTROL_IP is the IP of primary interface of control node i.e.
+ ## eth0, eno1. It is used by collectd monitor to communicate with
+ ## sample inspector.
+ ## @TODO (umar) see if mgmt IP of control is a better option. Also
+ ## primary interface may not be the right option
+ CONTROL_IP="$(ip a | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | sed -n 1p)"
+ #CONTROL_IP=192.168.98.6
+
+ echo "
+Hostname \"$COMPUTE_HOST\"
+FQDNLookup false
+Interval 1
+MaxReadInterval 2
+
+<LoadPlugin python>
+ Globals true
+</LoadPlugin>
+LoadPlugin ovs_events
+LoadPlugin logfile
+
+<Plugin logfile>
+ File \"/var/log/collectd.log\"
+ Timestamp true
+ LogLevel \"info\"
+</Plugin>
+
+<Plugin python>
+ ModulePath \"/home/$COMPUTE_USER\"
+ LogTraces true
+ Interactive false
+ Import \"collectd_plugin\"
+ <Module \"collectd_plugin\">
+ control_ip \"$CONTROL_IP\"
+ compute_ip \"$COMPUTE_IP\"
+ compute_host \"$COMPUTE_HOST\"
+ compute_user \"$COMPUTE_USER\"
+ inspector_type \"$INSPECTOR_TYPE\"
+ os_auth_url \"$OS_AUTH_URL\"
+ os_username \"$OS_USERNAME\"
+ os_password \"$OS_PASSWORD\"
+ os_project_name \"$OS_PROJECT_NAME\"
+ os_user_domain_name \"$OS_USER_DOMAIN_NAME\"
+ os_user_domain_id \"$OS_USER_DOMAIN_ID\"
+ os_project_domain_name \"$OS_PROJECT_DOMAIN_NAME\"
+ os_project_domain_id \"$OS_PROJECT_DOMAIN_ID\"
+ </Module>
+</Plugin>
+
+<Plugin ovs_events>
+ Port 6640
+ Socket \"/var/run/openvswitch/db.sock\"
+ Interfaces \"@INTERFACE_NAME@\"
+ SendNotification true
+ DispatchValues false
+</Plugin>
+
+" > $TOP_DIR/lib/monitors/collectd.conf
+
+ scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd.conf $COMPUTE_USER@$COMPUTE_IP:
+ ## @TODO (umar) Always assuming that the interface is assigned an IP if
+ ## interface name is not provided. See if there is a better approach
+ ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "
+ if [ -n \"$INTERFACE_NAME\" ]; then
+ dev=$INTERFACE_NAME
+ else
+ dev=\$(sudo ip a | awk '/ $COMPUTE_IP\//{print \$NF}')
+ fi
+ sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \$collectd_conf ]; then
+ sudo cp \$collectd_conf \${collectd_conf}-doctor-saved
+ else
+ sudo touch \${collectd_conf}-doctor-created
+ fi
+ sudo mv collectd.conf /opt/collectd/etc/collectd.conf"
+
+ scp $ssh_opts_cpu $TOP_DIR/lib/monitors/collectd/collectd_plugin.py $COMPUTE_USER@$COMPUTE_IP:collectd_plugin.py
+ ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "sudo pkill collectd
+ sudo /opt/collectd/sbin/collectd"
+}
+
+function stop_monitor_collectd {
+ ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'sudo pkill collectd'
+}
+
+function cleanup_monitor_collectd {
+ ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" "
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \"\${collectd_conf}-doctor-created\" ]; then
+ sudo rm \"\${collectd_conf}-doctor-created\"
+ sudo rm \$collectd_conf
+ elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then
+ sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf
+ sudo rm \"\${collectd_conf}-doctor-saved\"
+ fi"
+
+ rm $TOP_DIR/lib/monitors/collectd.conf
+}
diff --git a/tests/lib/monitors/collectd/collectd_plugin.py b/tests/lib/monitors/collectd/collectd_plugin.py
new file mode 100644
index 00000000..70fcf26e
--- /dev/null
+++ b/tests/lib/monitors/collectd/collectd_plugin.py
@@ -0,0 +1,167 @@
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import collectd
+import sys
+from netifaces import interfaces, ifaddresses, AF_INET
+from datetime import datetime
+import json
+import requests
+import time
+from requests.exceptions import ConnectionError
+
+from keystoneauth1 import loading
+from keystoneauth1 import session
+from congressclient.v1 import client
+
+
+def write_debug(str_write, write_type, compute_user):
+ file_name = ('/home/%s/monitor.log' % compute_user)
+ file_tmp = open(file_name, write_type)
+ file_tmp.write( "%s" % str_write)
+ file_tmp.close()
+
+
+class DoctorMonitorCollectd(object):
+ def __init__(self):
+ self.control_ip = ''
+ self.compute_user = ''
+ self.compute_ip = ''
+ self.host_name = ''
+ self.inspector_type = ''
+ self.inspector_url = ''
+ self.os_auth_url = ''
+ self.os_username = ''
+ self.os_password = ''
+ self.os_project_name = ''
+ self.os_user_domain_name = ''
+ self.os_user_domain_id = ''
+ self.os_project_domain_name = ''
+ self.os_project_domain_id = ''
+ self.sess = ''
+ self.auth = ''
+ self.inspector_notified = 0
+ self.start_notifications = 0
+ self.monitor_type = 'sample'
+
+ def config_func(self, config):
+ for node in config.children:
+ key = node.key.lower()
+ val = node.values[0]
+
+ if key == 'compute_host':
+ self.host_name = val
+ elif key == 'control_ip':
+ self.control_ip = val
+ elif key == 'compute_ip':
+ self.compute_ip = val
+ elif key == 'compute_user':
+ self.compute_user = val
+ elif key == 'inspector_type':
+ self.inspector_type = val
+ elif key == 'os_auth_url':
+ self.os_auth_url = val
+ elif key == 'os_username':
+ self.os_username = val
+ elif key == 'os_password':
+ self.os_password = val
+ elif key == 'os_project_name':
+ self.os_project_name = val
+ elif key == 'os_user_domain_name':
+ self.os_user_domain_name = val
+ elif key == 'os_user_domain_id':
+ self.os_user_domain_id = val
+ elif key == 'os_project_domain_name':
+ self.os_project_domain_name = val
+ elif key == 'os_project_domain_id':
+ self.os_project_domain_id = val
+ else:
+ collectd.info('Unknown config key "%s"' % key)
+
+ def init_collectd(self):
+ write_debug("Compute node collectd monitor start at %s\n\n" % datetime.now().isoformat(), "w", self.compute_user)
+
+ if self.inspector_type == 'sample':
+ self.inspector_url = ('http://%s:12345/events' % self.control_ip)
+ elif self.inspector_type == 'congress':
+ loader = loading.get_plugin_loader('password')
+ self.auth = loader.load_from_options(auth_url=self.os_auth_url,
+ username=self.os_username,
+ password=self.os_password,
+ project_name=self.os_project_name,
+ user_domain_name=self.os_user_domain_name,
+ user_domain_id=self.os_user_domain_id,
+ project_domain_name=self.os_project_domain_name,
+ project_domain_id=self.os_project_domain_id)
+ self.sess=session.Session(auth=self.auth)
+ congress = client.Client(session=self.sess, service_type='policy')
+ ds = congress.list_datasources()['results']
+ doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+ None)
+
+ congress_endpoint = congress.httpclient.get_endpoint(auth=self.auth)
+ self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' %
+ (congress_endpoint, doctor_ds['id']))
+ else:
+ sys.exit()
+ self.start_notifications = 1
+
+
+ def notify_inspector(self):
+ event_type = "compute.host.down"
+ payload = [
+ {
+ 'id': ("monitor_%s_id1" % self.monitor_type),
+ 'time': datetime.now().isoformat(),
+ 'type': event_type,
+ 'details': {
+ 'hostname': self.host_name,
+ 'status': 'down',
+ 'monitor': ("monitor_%s" % self.monitor_type),
+ 'monitor_event_id': ("monitor_%s_event1" % self.monitor_type)
+ },
+ },
+ ]
+ data = json.dumps(payload)
+ self.inspector_notified = 1
+
+ if self.inspector_type == 'sample':
+ headers = {'content-type': 'application/json'}
+ try:
+ requests.post(self.inspector_url, data=data, headers=headers)
+ except ConnectionError as err:
+ print err
+ elif self.inspector_type == 'congress':
+ # TODO(umar) enhance for token expiry case
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ 'X-Auth-Token': self.sess.get_token()
+ }
+ requests.put(self.inspector_url, data=data, headers=headers)
+
+
+ def handle_notif(self, notification, data=None):
+ if (notification.severity == collectd.NOTIF_FAILURE or
+ notification.severity == collectd.NOTIF_WARNING):
+ if (self.start_notifications == 1 and self.inspector_notified == 0):
+ write_debug("Received down notification: doctor monitor detected at %s\n" % time.time(), "a", self.compute_user)
+ self.notify_inspector()
+
+ elif notification.severity == collectd.NOTIF_OKAY:
+ collectd.info("Interface status: UP again %s\n" % time.time())
+ else:
+ collectd.info("Unknown notification severity %s\n" % notification.severity)
+
+
+monitor = DoctorMonitorCollectd()
+
+collectd.register_config(monitor.config_func)
+collectd.register_init(monitor.init_collectd)
+collectd.register_notification(monitor.handle_notif)
diff --git a/tests/monitor.py b/tests/lib/monitors/sample/monitor.py
index 7450c534..7450c534 100644
--- a/tests/monitor.py
+++ b/tests/lib/monitors/sample/monitor.py
diff --git a/tests/lib/monitors/sample/sample b/tests/lib/monitors/sample/sample
new file mode 100644
index 00000000..1d310333
--- /dev/null
+++ b/tests/lib/monitors/sample/sample
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+function start_monitor_sample {
+ cp $TOP_DIR/lib/monitors/sample/monitor.py $TOP_DIR/monitor.py
+ pgrep -f "python monitor.py" && return 0
+ sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \
+ > monitor.log 2>&1 &
+}
+
+function stop_monitor_sample {
+ pgrep -f "python monitor.py" || return 0
+ sudo kill $(pgrep -f "python monitor.py")
+}
+
+function cleanup_monitor_sample {
+ rm monitor.py
+ return
+}
diff --git a/tests/logger.py b/tests/logger.py
index 72043ab3..021389d9 100644
--- a/tests/logger.py
+++ b/tests/logger.py
@@ -21,6 +21,7 @@ class Logger(object):
CI_DEBUG = os.getenv('CI_DEBUG')
+ logging.basicConfig(filemode='w')
self.logger = logging.getLogger(logger_name)
self.logger.propagate = 0
self.logger.setLevel(logging.DEBUG)
@@ -41,7 +42,5 @@ class Logger(object):
file_handler.setLevel(logging.DEBUG)
self.logger.addHandler(file_handler)
-
def getLogger(self):
return self.logger
-
diff --git a/tests/main.py b/tests/main.py
index 50e0821b..db2fafd9 100644
--- a/tests/main.py
+++ b/tests/main.py
@@ -6,11 +6,21 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+import os
+from os.path import isfile, join
import sys
+from alarm import Alarm
import config
+from consumer import get_consumer
from image import Image
+from instance import Instance
+from inspector import get_inspector
+from installer import get_installer
import logger as doctor_log
+from user import User
+from network import Network
+from monitor import get_monitor
LOG = doctor_log.Logger('doctor').getLogger()
@@ -20,38 +30,82 @@ class DoctorTest(object):
def __init__(self, conf):
self.conf = conf
- self.image = Image(self.conf)
+ self.image = Image(self.conf, LOG)
+ self.user = User(self.conf, LOG)
+ self.network = Network(self.conf, LOG)
+ self.instance = Instance(self.conf, LOG)
+ self.alarm = Alarm(self.conf, LOG)
+ self.inspector = get_inspector(self.conf, LOG)
+ self.monitor = get_monitor(self.conf,
+ self.inspector.get_inspector_url(),
+ LOG)
+ self.consumer = get_consumer(self.conf, LOG)
+ self.installer = get_installer(self.conf, LOG)
+
+ def setup(self):
+ # prepare the cloud env
+ self.installer.setup()
+
+ # preparing VM image...
+ self.image.create()
+
+ # creating test user...
+ self.user.create()
+ self.user.update_quota()
+
+ # creating VM...
+ self.network.create()
+ self.instance.create()
+ self.instance.wait_for_vm_launch()
+
+ # creating alarm...
+ self.alarm.create()
+
+ # starting doctor sample components...
+ self.inspector.start()
+ self.monitor.start()
+ self.consumer.start()
def run(self):
"""run doctor test"""
try:
LOG.info('doctor test starting.......')
- # prepare the cloud env
- # preparing VM image...
- self.image.create()
-
- # creating test user...
-
- # creating VM...
-
- # creating alarm...
-
- # starting doctor sample components...
+ self.setup()
# injecting host failure...
+ # NOTE (umar) add INTERFACE_NAME logic to host injection
# verify the test results
+ # NOTE (umar) copy remote monitor.log file when monitor=collectd
+
except Exception as e:
LOG.error('doctor test failed, Exception=%s' % e)
sys.exit(1)
finally:
- self.image.delete()
+ self.cleanup()
+
+ def cleanup(self):
+ self.alarm.delete()
+ self.instance.delete()
+ self.network.delete()
+ self.image.delete()
+ self.inspector.stop()
+ self.user.delete()
+ self.monitor.stop()
+ self.consumer.stop()
+ self.installer.cleanup()
def main():
"""doctor main"""
- conf = config.prepare_conf()
+ doctor_root_dir = os.path.dirname(sys.path[0])
+ config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/')
+ config_files = [join(config_file_dir, f) for f in os.listdir(config_file_dir)
+ if isfile(join(config_file_dir, f))]
+
+ conf = config.prepare_conf(args=sys.argv[1:],
+ config_files=config_files)
doctor = DoctorTest(conf)
doctor.run()
diff --git a/tests/monitor/__init__.py b/tests/monitor/__init__.py
new file mode 100644
index 00000000..e268907f
--- /dev/null
+++ b/tests/monitor/__init__.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+from oslo_utils import importutils
+
+OPTS = [
+ cfg.StrOpt('type',
+ default='sample',
+ choices=['sample', 'collectd'],
+ help='the type of doctor monitor component',
+ required=True),
+]
+
+
+_monitor_name_class_mapping = {
+ 'sample': 'monitor.sample.SampleMonitor',
+ 'collectd': 'monitor.collectd.CollectdMonitor'
+}
+
+def get_monitor(conf, inspector_url, log):
+ monitor_class = _monitor_name_class_mapping.get(conf.monitor.type)
+ return importutils.import_object(monitor_class, conf,
+ inspector_url, log)
diff --git a/tests/monitor/base.py b/tests/monitor/base.py
new file mode 100644
index 00000000..ccb647cf
--- /dev/null
+++ b/tests/monitor/base.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseMonitor(object):
+ """Monitor computer fault and report error to the inspector"""
+ def __init__(self, conf, inspector_url, log):
+ self.conf = conf
+ self.log = log
+ self.inspector_url = inspector_url
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass
diff --git a/tests/monitor/collectd.py b/tests/monitor/collectd.py
new file mode 100644
index 00000000..f7a4f442
--- /dev/null
+++ b/tests/monitor/collectd.py
@@ -0,0 +1,145 @@
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+import socket
+import getpass
+import sys
+
+from identity_auth import get_session
+from os_clients import nova_client
+from monitor.base import BaseMonitor
+
+
+class CollectdMonitor(BaseMonitor):
+ def __init__(self, conf, inspector_url, log):
+ super(CollectdMonitor, self).__init__(conf, inspector_url, log)
+ self.top_dir = os.path.dirname(sys.path[0])
+ self.session = get_session()
+ self.nova = nova_client(conf.nova_version, self.session)
+ self.compute_hosts = self.nova.hypervisors.list(detailed=True)
+ for host in self.compute_hosts:
+ host_dict = host.__dict__
+ self.compute_host = host_dict['hypervisor_hostname']
+ self.compute_ip = host_dict['host_ip']
+ tmp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ tmp_sock.connect(("8.8.8.8", 80))
+
+ ## control_ip is the IP of primary interface of control node i.e.
+ ## eth0, eno1. It is used by collectd monitor to communicate with
+ ## sample inspector.
+ ## TODO (umar) see if mgmt IP of control is a better option. Also
+ ## primary interface may not be the right option
+ self.control_ip = tmp_sock.getsockname()[0]
+ self.compute_user = getpass.getuser()
+ self.interface_name = os.environ.get('INTERFACE_NAME') or ''
+ self.inspector_type = os.environ.get('INSPECTOR_TYPE', 'sample')
+ self.auth_url = os.environ.get('OS_AUTH_URL')
+ self.username = os.environ.get('OS_USERNAME')
+ self.password = os.environ.get('OS_PASSWORD')
+ self.project_name = os.environ.get('OS_PROJECT_NAME')
+ self.user_domain_name = os.environ.get('OS_USER_DOMAIN_NAME') or 'default'
+ self.user_domain_id = os.environ.get('OS_USER_DOMAIN_ID')
+ self.project_domain_name = os.environ.get('OS_PROJECT_DOMAIN_NAME') or 'default'
+ self.project_domain_id = os.environ.get('OS_PROJECT_DOMAIN_ID')
+ self.ssh_opts_cpu = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+
+ def start(self):
+ self.log.info("Collectd monitor start.........")
+ f = open("%s/tests/collectd.conf" % self.top_dir, 'w')
+ collectd_conf_file = """
+Hostname %s
+FQDNLookup false
+Interval 1
+MaxReadInterval 2
+
+<LoadPlugin python>
+Globals true
+</LoadPlugin>
+LoadPlugin ovs_events
+LoadPlugin logfile
+
+<Plugin logfile>
+ File \"/var/log/collectd.log\"
+ Timestamp true
+ LogLevel \"info\"
+</Plugin>
+
+<Plugin python>
+ ModulePath \"/home/%s\"
+ LogTraces true
+ Interactive false
+ Import \"collectd_plugin\"
+ <Module \"collectd_plugin\">
+ control_ip \"%s\"
+ compute_ip \"%s\"
+ compute_host \"%s\"
+ compute_user \"%s\"
+ inspector_type \"%s\"
+ os_auth_url \"%s\"
+ os_username \"%s\"
+ os_password \"%s\"
+ os_project_name \"%s\"
+ os_user_domain_name \"%s\"
+ os_user_domain_id \"%s\"
+ os_project_domain_name \"%s\"
+ os_project_domain_id \"%s\"
+ </Module>
+</Plugin>
+
+<Plugin ovs_events>
+ Port 6640
+ Socket \"/var/run/openvswitch/db.sock\"
+ Interfaces \"@INTERFACE_NAME@\"
+ SendNotification true
+ DispatchValues false
+</Plugin>
+ """ % (self.compute_host, self.compute_user, self.control_ip, self.compute_ip, self.compute_host, self.compute_user,
+ self.inspector_type, self.auth_url, self.username, self.password, self.project_name, self.user_domain_name,
+ self.user_domain_id, self.project_domain_name, self.project_domain_id)
+ f.write(collectd_conf_file)
+ f.close()
+
+ os.system(" scp %s %s/tests/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+ self.log.info("after first scp")
+ ## @TODO (umar) Always assuming that the interface is assigned an IP if
+ ## interface name is not provided. See if there is a better approach
+ os.system(""" ssh %s %s@%s \"if [ -n \"%s\" ]; then
+ dev=%s
+ else
+ dev=\$(sudo ip a | awk '/ %s\//{print \$NF}')
+ fi
+ sed -i -e \"s/@INTERFACE_NAME@/\$dev/\" collectd.conf
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \$collectd_conf ]; then
+ sudo cp \$collectd_conf \${collectd_conf}-doctor-saved
+ else
+ sudo touch \${collectd_conf}-doctor-created
+ fi
+ sudo mv collectd.conf /opt/collectd/etc/collectd.conf\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip, self.interface_name, self.interface_name, self.compute_ip))
+ self.log.info("after first ssh")
+ os.system(" scp %s %s/tests/lib/monitors/collectd/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+ self.log.info("after sec scp")
+ os.system(" ssh %s %s@%s \"sudo pkill collectd; sudo /opt/collectd/sbin/collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+ self.log.info("after sec ssh")
+
+ def stop(self):
+ os.system(" ssh %s %s@%s \"sudo pkill collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+
+ def cleanup(self):
+ os.system(""" ssh %s %s@%s \"
+ collectd_conf=/opt/collectd/etc/collectd.conf
+ if [ -e \"\${collectd_conf}-doctor-created\" ]; then
+ sudo rm \"\${collectd_conf}-doctor-created\"
+ sudo rm \$collectd_conf
+ elif [ -e \"\${collectd_conf}-doctor-saved\" ]; then
+ sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf
+ sudo rm \"\${collectd_conf}-doctor-saved\"
+ fi\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
+ os.remove("%s/tests/collectd.conf" % self.top_dir)
diff --git a/tests/monitor/sample.py b/tests/monitor/sample.py
new file mode 100644
index 00000000..1333a2ec
--- /dev/null
+++ b/tests/monitor/sample.py
@@ -0,0 +1,115 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from datetime import datetime
+import json
+import requests
+import socket
+from threading import Thread
+import time
+
+from identity_auth import get_session
+from os_clients import nova_client
+from monitor.base import BaseMonitor
+
+
+class SampleMonitor(BaseMonitor):
+ event_type = "compute.host.down"
+
+ def __init__(self, conf, inspector_url, log):
+ super(SampleMonitor, self).__init__(conf, inspector_url, log)
+ self.session = get_session()
+ self.nova = nova_client(conf.nova_version, self.session)
+ self.hosts = self.nova.hypervisors.list(detailed=True)
+ self.pingers = []
+
+ def start(self):
+ self.log.info('sample monitor start......')
+ for host in self.hosts:
+ host_dict = host.__dict__
+ host_name = host_dict['hypervisor_hostname']
+ host_ip = host_dict['host_ip']
+ pinger = Pinger(host_name, host_ip, self, self.log)
+ pinger.start()
+ self.pingers.append(pinger)
+
+ def stop(self):
+ self.log.info('sample monitor stop......')
+ for pinger in self.pingers:
+ pinger.stop()
+ pinger.join()
+ del self.pingers
+
+ def report_error(self, hostname):
+ self.log.info('sample monitor report error......')
+ data = [
+ {
+ 'id': 'monitor_sample_id1',
+ 'time': datetime.now().isoformat(),
+ 'type': self.event_type,
+ 'details': {
+ 'hostname': hostname,
+ 'status': 'down',
+ 'monitor': 'monitor_sample',
+ 'monitor_event_id': 'monitor_sample_event1'
+ },
+ },
+ ]
+
+ auth_token = self.session.get_token() if \
+ self.conf.inspector.type != 'sample' else None
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ 'X-Auth-Token': auth_token,
+ }
+
+ url = '%s%s' % (self.inspector_url, 'events') \
+ if self.inspector_url.endswith('/') else \
+ '%s%s' % (self.inspector_url, '/events')
+ requests.put(url, data=json.dumps(data), headers=headers)
+
+
+class Pinger(Thread):
+ interval = 0.1 # second
+ timeout = 0.1 # second
+ ICMP_ECHO_MESSAGE = bytes([0x08, 0x00, 0xf7, 0xff, 0x00, 0x00, 0x00, 0x00])
+
+ def __init__(self, host_name, host_ip, monitor, log):
+ Thread.__init__(self)
+ self.monitor = monitor
+ self.hostname = host_name
+ self.ip_addr = host_ip or socket.gethostbyname(self.hostname)
+ self.log = log
+ self._stopped = False
+
+ def run(self):
+ self.log.info("Starting Pinger host_name(%s), host_ip(%s)"
+ % (self.hostname, self.ip_addr))
+
+ sock = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_ICMP)
+ sock.settimeout(self.timeout)
+ while True:
+ if self._stopped:
+ return
+ try:
+ sock.sendto(self.ICMP_ECHO_MESSAGE, (self.ip_addr, 0))
+ sock.recv(4096)
+ except socket.timeout:
+ self.log.info("doctor monitor detected at %s" % time.time())
+ self.monitor.report_error(self.hostname)
+ self.log.info("ping timeout, quit monitoring...")
+ self._stopped = True
+ return
+ time.sleep(self.interval)
+
+ def stop(self):
+ self.log.info("Stopping Pinger host_name(%s), host_ip(%s)"
+ % (self.hostname, self.ip_addr))
+ self._stopped = True
diff --git a/tests/network.py b/tests/network.py
new file mode 100644
index 00000000..da7ad09d
--- /dev/null
+++ b/tests/network.py
@@ -0,0 +1,68 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from oslo_config import cfg
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import neutron_client
+
+
+OPTS = [
+ cfg.StrOpt('net_name',
+ default='doctor_net',
+ help='the name of test net',
+ required=True),
+ cfg.StrOpt('net_cidr',
+ default='192.168.168.0/24',
+ help='the cidr of test subnet',
+ required=True),
+]
+
+
+class Network(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.auth = get_identity_auth(username=self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ project=self.conf.doctor_project)
+ self.neutron = neutron_client(get_session(auth=self.auth))
+ self.net = None
+ self.subnet = None
+
+ def create(self):
+ self.log.info('network create start.......')
+ net_name = self.conf.net_name
+ networks = self.neutron.list_networks(name=net_name)['networks']
+ self.net = networks[0] if networks \
+ else self.neutron.create_network(
+ {'network': {'name': net_name}})['network']
+ self.log.info('network create end.......')
+
+ self.log.info('subnet create start.......')
+ subnets = self.neutron.list_subnets(network_id=self.net['id'])['subnets']
+ subnet_param = {'name': net_name, 'network_id': self.net['id'],
+ 'cidr': self.conf.net_cidr, 'ip_version': 4,
+ 'enable_dhcp': False}
+ self.subnet = subnets[0] if subnets \
+ else self.neutron.create_subnet(
+ {'subnet': subnet_param})['subnet']
+ self.log.info('subnet create end.......')
+
+ def delete(self):
+ self.log.info('subnet delete start.......')
+ if self.subnet:
+ self.neutron.delete_subnet(self.subnet['id'])
+ self.log.info('subnet delete end.......')
+
+ self.log.info('network delete start.......')
+ if self.net:
+ self.neutron.delete_network(self.net['id'])
+ self.log.info('network delete end.......')
diff --git a/tests/os_clients.py b/tests/os_clients.py
index 2eb406e0..44fa3aad 100644
--- a/tests/os_clients.py
+++ b/tests/os_clients.py
@@ -8,14 +8,43 @@
##############################################################################
from oslo_config import cfg
+import aodhclient.client as aodhclient
+from congressclient.v1 import client as congressclient
import glanceclient.client as glanceclient
+from keystoneclient.v2_0 import client as ks_client
+from neutronclient.v2_0 import client as neutronclient
+import novaclient.client as novaclient
OPTS = [
cfg.StrOpt('glance_version', default='2', help='glance version'),
+ cfg.StrOpt('nova_version', default='2.34', help='Nova version'),
+ cfg.StrOpt('aodh_version', default='2', help='aodh version'),
]
def glance_client(version, session):
return glanceclient.Client(version=version,
session=session)
+
+
+def keystone_client(session):
+ return ks_client.Client(session=session)
+
+
+def nova_client(version, session):
+ return novaclient.Client(version=version,
+ session=session)
+
+
+def neutron_client(session):
+ return neutronclient.Client(session=session)
+
+
+def aodh_client(version, session):
+ return aodhclient.Client(version, session=session)
+
+
+def congress_client(session):
+ return congressclient.Client(session=session,
+ service_type='policy')
diff --git a/tests/run.sh b/tests/run.sh
index fda1e753..dceb0614 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -29,7 +29,7 @@ DOCTOR_USER=doctor
DOCTOR_PW=doctor
DOCTOR_PROJECT=doctor
DOCTOR_ROLE=_member_
-PROFILER_TYPE=${PROFILER_TYPE:-none}
+PROFILER_TYPE=${PROFILER_TYPE:-poc}
PYTHON_ENABLE=${PYTHON_ENABLE:-false}
TOP_DIR=$(cd $(dirname "$0") && pwd)
@@ -48,7 +48,7 @@ as_admin_user="--os-username admin --os-project-name $DOCTOR_PROJECT
get_compute_host_info() {
# get computer host info which first VM boot in as admin user
COMPUTE_HOST=$(openstack $as_admin_user server show ${VM_BASENAME}1 |
- grep "OS-EXT-SRV-ATTR:host" | awk '{ print $4 }')
+ grep "OS-EXT-SRV-ATTR:host " | awk '{ print $4 }')
compute_host_in_undercloud=${COMPUTE_HOST%%.*}
die_if_not_set $LINENO COMPUTE_HOST "Failed to get compute hostname"
@@ -212,17 +212,6 @@ create_alarm() {
done
}
-start_monitor() {
- pgrep -f "python monitor.py" && return 0
- sudo -E python monitor.py "$COMPUTE_HOST" "$COMPUTE_IP" "$INSPECTOR_TYPE" \
- > monitor.log 2>&1 &
-}
-
-stop_monitor() {
- pgrep -f "python monitor.py" || return 0
- sudo kill $(pgrep -f "python monitor.py")
-}
-
start_consumer() {
pgrep -f "python consumer.py" && return 0
python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1 &
@@ -294,8 +283,12 @@ inject_failure() {
echo "disabling network of compute host [$COMPUTE_HOST] for 3 mins..."
cat > disable_network.sh << 'END_TXT'
#!/bin/bash -x
-dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}')
sleep 1
+if [ -n "@INTERFACE_NAME@" ]; then
+ dev=@INTERFACE_NAME@
+else
+ dev=$(sudo ip a | awk '/ @COMPUTE_IP@\//{print $NF}')
+fi
sudo ip link set $dev down
echo "doctor set link down at" $(date "+%s.%N")
sleep 180
@@ -303,6 +296,7 @@ sudo ip link set $dev up
sleep 1
END_TXT
sed -i -e "s/@COMPUTE_IP@/$COMPUTE_IP/" disable_network.sh
+ sed -i -e "s/@INTERFACE_NAME@/$INTERFACE_NAME/" disable_network.sh
chmod +x disable_network.sh
scp $ssh_opts_cpu disable_network.sh "$COMPUTE_USER@$COMPUTE_IP:"
ssh $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP" 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
@@ -327,8 +321,11 @@ calculate_notification_time() {
wait_consumer 60
#keep 'at' as the last keyword just before the value, and
#use regex to get value instead of the fixed column
+ if [ ! -f monitor.log ]; then
+ scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:monitor.log" .
+ fi
detected=$(grep "doctor monitor detected at" monitor.log |\
- sed -e "s/^.* at //")
+ sed -e "s/^.* at //" | tail -1)
notified=$(grep "doctor consumer notified at" consumer.log |\
sed -e "s/^.* at //" | tail -1)
@@ -424,18 +421,18 @@ run_profiler() {
export DOCTOR_PROFILER_T09=$(python -c \
"print(int(($notified-$relative_start)*1000))")
- python profiler-poc.py >doctor_profiler.log 2>&1
+ python profiler-poc.py > doctor_profiler.log 2>&1
fi
}
cleanup() {
set +e
echo "cleanup..."
- stop_monitor
stop_inspector
stop_consumer
unset_forced_down_hosts
+ stop_monitor
collect_logs
vms=$(openstack $as_doctor_user server list)
@@ -467,6 +464,7 @@ cleanup() {
cleanup_installer
cleanup_inspector
+ cleanup_monitor
# NOTE: Temporal log printer.
for f in $(find . -name '*.log')
@@ -478,9 +476,23 @@ cleanup() {
done
}
+setup_python_packages() {
+ sudo pip install flask==0.10.1
+ command -v openstack || sudo pip install python-openstackclient==2.3.0
+ command -v ceilometer || sudo pip install python-ceilometerclient==2.6.2
+ command -v congress || sudo pip install python-congressclient==1.5.0
+}
+
# Main process
-if $PYTHON_ENABLE; then
+if [[ $PYTHON_ENABLE == [Tt]rue ]]; then
+ which tox || sudo pip install tox
+ if [ -f /usr/bin/apt-get ]; then
+ sudo apt-get install -y python3-dev
+ elif [ -f /usr/bin/yum ] ; then
+ sudo yum install -y python3-devel
+ fi
+
cd $TOP_DIR
echo "executing tox..."
tox
@@ -492,9 +504,14 @@ git log --oneline -1 || true # ignore even you don't have git installed
trap cleanup EXIT
+setup_python_packages
+
source $TOP_DIR/functions-common
source $TOP_DIR/lib/installer
source $TOP_DIR/lib/inspector
+source $TOP_DIR/lib/monitor
+
+rm -f *.log
setup_installer
@@ -524,8 +541,8 @@ echo "injecting host failure..."
inject_failure
check_host_status "(DOWN|UNKNOWN)" 60
-calculate_notification_time
unset_forced_down_hosts
+calculate_notification_time
collect_logs
run_profiler
diff --git a/tests/user.py b/tests/user.py
new file mode 100644
index 00000000..b21bd1a8
--- /dev/null
+++ b/tests/user.py
@@ -0,0 +1,163 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+
+from identity_auth import get_session
+from os_clients import keystone_client
+from os_clients import nova_client
+
+
+OPTS = [
+ cfg.StrOpt('doctor_user',
+ default='doctor',
+ help='the name of test user',
+ required=True),
+ cfg.StrOpt('doctor_passwd',
+ default='doctor',
+ help='the password of test user',
+ required=True),
+ cfg.StrOpt('doctor_project',
+ default='doctor',
+ help='the name of test project',
+ required=True),
+ cfg.StrOpt('doctor_role',
+ default='_member_',
+ help='the role of test user',
+ required=True),
+ cfg.IntOpt('quota_instances',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the quota of instances in test user',
+ required=True),
+ cfg.IntOpt('quota_cores',
+ default=os.environ.get('VM_COUNT', 1),
+ help='the quota of cores in test user',
+ required=True),
+]
+
+
+class User(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+ self.keystone = \
+ keystone_client(get_session())
+ self.nova = \
+ nova_client(conf.nova_version, get_session())
+ self.users = {}
+ self.projects = {}
+ self.roles = {}
+ self.roles_for_user = {}
+ self.roles_for_admin = {}
+
+ def create(self):
+ """create test user, project and etc"""
+ self.log.info('user create start......')
+
+ self._create_project()
+ self._create_user()
+ self._create_role()
+ self._add_user_role_in_project(is_admin=False)
+ self._add_user_role_in_project(is_admin=True)
+
+ self.log.info('user create end......')
+
+ def _create_project(self):
+ """create test project"""
+ self.projects = {project.name: project
+ for project in self.keystone.tenants.list()}
+ if self.conf.doctor_project not in self.projects:
+ test_project = \
+ self.keystone.tenants.create(self.conf.doctor_project)
+ self.projects[test_project.name] = test_project
+
+ def _create_user(self):
+ """create test user"""
+ project = self.projects.get(self.conf.doctor_project)
+ self.users = {user.name: user for user in self.keystone.users.list()}
+ if self.conf.doctor_user not in self.users:
+ test_user = self.keystone.users.create(
+ self.conf.doctor_user,
+ password=self.conf.doctor_passwd,
+ tenant_id=project.id)
+ self.users[test_user.name] = test_user
+
+ def _create_role(self):
+ """create test role"""
+ self.roles = {role.name: role for role in self.keystone.roles.list()}
+ if self.conf.doctor_role not in self.roles:
+ test_role = self.keystone.roles.create(self.conf.doctor_role)
+ self.roles[test_role.name] = test_role
+
+ def _add_user_role_in_project(self, is_admin=False):
+ """add test user with test role in test project"""
+ project = self.projects.get(self.conf.doctor_project)
+
+ user_name = 'admin' if is_admin else self.conf.doctor_user
+ user = self.users.get(user_name)
+
+ role_name = 'admin' if is_admin else self.conf.doctor_role
+ role = self.roles.get(role_name)
+
+ roles_for_user = self.roles_for_admin \
+ if is_admin else self.roles_for_user
+
+ roles_for_user = \
+ {role.name: role for role in
+ self.keystone.roles.roles_for_user(user, tenant=project)}
+ if role_name not in roles_for_user:
+ self.keystone.roles.add_user_role(user, role, tenant=project)
+ roles_for_user[role_name] = role
+
+ def delete(self):
+ """delete the test user, project and role"""
+ self.log.info('user delete start......')
+
+ project = self.projects.get(self.conf.doctor_project)
+ user = self.users.get(self.conf.doctor_user)
+ role = self.roles.get(self.conf.doctor_role)
+
+ if project:
+ if 'admin' in self.roles_for_admin:
+ self.keystone.roles.remove_user_role(
+ self.users['admin'],
+ self.roles['admin'],
+ tenant=project)
+
+ if user:
+ if role and self.conf.doctor_role in self.roles_for_user:
+ self.keystone.roles.remove_user_role(
+ user, role, tenant=project)
+ self.keystone.roles.delete(role)
+ self.keystone.users.delete(user)
+
+ self.keystone.tenants.delete(project)
+ self.log.info('user delete end......')
+
+ def update_quota(self):
+ self.log.info('user quota update start......')
+ project = self.projects.get(self.conf.doctor_project)
+ user = self.users.get(self.conf.doctor_user)
+
+ if project and user:
+ self.quota = self.nova.quotas.get(project.id,
+ user_id=user.id)
+ if self.conf.quota_instances > self.quota.instances:
+ self.nova.quotas.update(project.id,
+ instances=self.conf.quota_instances,
+ user_id=user.id)
+ if self.conf.quota_cores > self.quota.cores:
+ self.nova.quotas.update(project.id,
+ cores=self.conf.quota_cores,
+ user_id=user.id)
+ self.log.info('user quota update end......')
+ else:
+ raise Exception('No project or role for update quota')
diff --git a/tests/utils.py b/tests/utils.py
new file mode 100644
index 00000000..41e22353
--- /dev/null
+++ b/tests/utils.py
@@ -0,0 +1,78 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import json
+import os
+import paramiko
+
+
+def load_json_file(full_path):
+ """Loads JSON from file
+ :param target_filename:
+ :return:
+ """
+ if not os.path.isfile(full_path):
+ raise Exception('File(%s) does not exist' % full_path)
+
+ with open(full_path, 'r') as file:
+ return json.load(file)
+
+
+def write_json_file(full_path, data):
+ """write JSON from file
+ :param target_filename:
+ :return:
+ """
+
+ with open(full_path, 'w+') as file:
+ file.write(json.dumps(data))
+
+
+class SSHClient(object):
+ def __init__(self, ip, username, password=None, pkey=None,
+ key_filename=None, log=None, look_for_keys=False,
+ allow_agent=False):
+ self.client = paramiko.SSHClient()
+ self.client.load_system_host_keys()
+ self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ self.client.connect(ip, username=username, password=password,
+ pkey=pkey, key_filename=key_filename,
+ look_for_keys=look_for_keys,
+ allow_agent=allow_agent)
+ self.log = log
+
+ def __del__(self):
+ self.client.close()
+
+ def ssh(self, command):
+ if self.log:
+ self.log.debug("Executing: %s" % command)
+ stdin, stdout, stderr = self.client.exec_command(command)
+ ret = stdout.channel.recv_exit_status()
+ output = list()
+ for line in stdout.read().splitlines():
+ output.append(line.decode('utf-8'))
+ if ret:
+ if self.log:
+ self.log.debug("*** FAILED to run command %s (%s)" % (command, ret))
+ raise Exception(
+ "Unable to run \ncommand: %s\nret: %s"
+ % (command, ret))
+ if self.log:
+ self.log.debug("*** SUCCESSFULLY run command %s" % command)
+ return ret, output
+
+ def scp(self, source, dest, method='put'):
+ if self.log:
+ self.log.info("Copy %s -> %s" % (source, dest))
+ ftp = self.client.open_sftp()
+ if method == 'put':
+ ftp.put(source, dest)
+ elif method == 'get':
+ ftp.get(source, dest)
+ ftp.close()