summaryrefslogtreecommitdiffstats
path: root/tests/inspector
diff options
context:
space:
mode:
Diffstat (limited to 'tests/inspector')
-rw-r--r--tests/inspector/__init__.py40
-rw-r--r--tests/inspector/base.py30
-rw-r--r--tests/inspector/congress.py94
-rw-r--r--tests/inspector/sample.py151
4 files changed, 315 insertions, 0 deletions
diff --git a/tests/inspector/__init__.py b/tests/inspector/__init__.py
new file mode 100644
index 00000000..afba4800
--- /dev/null
+++ b/tests/inspector/__init__.py
@@ -0,0 +1,40 @@
+#############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+
+from oslo_config import cfg
+from oslo_utils import importutils
+
+
+OPTS = [
+ cfg.StrOpt('type',
+ default=os.environ.get('INSPECTOR_TYPE', 'sample'),
+ choices=['sample', 'congress', 'vitrage'],
+ help='the component of doctor inspector',
+ required=True),
+ cfg.StrOpt('ip',
+ default='127.0.0.1',
+ help='the host ip of inspector',
+ required=False),
+ cfg.StrOpt('port',
+ default='12345',
+ help='the port of default for inspector',
+ required=False),
+]
+
+
+_inspector_name_class_mapping = {
+ 'sample': 'inspector.sample.SampleInspector',
+ 'congress': 'inspector.congress.CongressInspector',
+}
+
+
+def get_inspector(conf, log):
+ inspector_class = _inspector_name_class_mapping[conf.inspector.type]
+ return importutils.import_object(inspector_class, conf, log)
diff --git a/tests/inspector/base.py b/tests/inspector/base.py
new file mode 100644
index 00000000..854f0695
--- /dev/null
+++ b/tests/inspector/base.py
@@ -0,0 +1,30 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseInspector(object):
+
+ def __init__(self, conf, log):
+ self.conf = conf
+ self.log = log
+
+ @abc.abstractmethod
+ def get_inspector_url(self):
+ pass
+
+ @abc.abstractmethod
+ def start(self):
+ pass
+
+ @abc.abstractmethod
+ def stop(self):
+ pass \ No newline at end of file
diff --git a/tests/inspector/congress.py b/tests/inspector/congress.py
new file mode 100644
index 00000000..ae295852
--- /dev/null
+++ b/tests/inspector/congress.py
@@ -0,0 +1,94 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import congress_client
+
+from inspector.base import BaseInspector
+
+
+class CongressInspector(BaseInspector):
+ nova_api_min_version = '2.11'
+ doctor_driver = 'doctor'
+ doctor_datasource = 'doctor'
+ policy = 'classification'
+ rules = {
+ 'host_down':
+ 'host_down(host) :- doctor:events(hostname=host, type="compute.host.down", status="down")',
+ 'active_instance_in_host':
+ 'active_instance_in_host(vmid, host) :- nova:servers(id=vmid, host_name=host, status="ACTIVE")',
+ 'host_force_down':
+ 'execute[nova:services.force_down(host, "nova-compute", "True")] :- host_down(host)',
+ 'error_vm_states':
+ 'execute[nova:servers.reset_state(vmid, "error")] :- host_down(host), active_instance_in_host(vmid, host)'
+ }
+
+ def __init__(self, conf, log):
+ super(CongressInspector, self).__init__(conf, log)
+ self.auth = get_identity_auth()
+ self.congress = congress_client(get_session(auth=self.auth))
+ self._init_driver_and_ds()
+ self.inspector_url = self.get_inspector_url()
+
+ def _init_driver_and_ds(self):
+ datasources = \
+ {ds['name']: ds for ds in self.congress.list_datasources()['results']}
+
+ # check nova_api version
+ nova_api_version = datasources['nova']['config'].get('api_version')
+ if nova_api_version and nova_api_version < self.nova_api_min_version:
+ raise Exception('Congress Nova datasource API version < nova_api_min_version(%s)'
+ % self.nova_api_min_version)
+
+ # create doctor datasource if it's not exist
+ if self.doctor_datasource not in datasources:
+ self.congress.create_datasource(
+ body={'driver': self.doctor_driver,
+ 'name': self.doctor_datasource})
+
+ # check whether doctor driver exist
+ drivers = \
+ {driver['id']: driver for driver in self.congress.list_drivers()['results']}
+ if self.doctor_driver not in drivers:
+ raise Exception('Do not support doctor driver in congress')
+
+ self.policy_rules = \
+ {rule['name']: rule for rule in
+ self.congress.list_policy_rules(self.policy)['results']}
+
+ def get_inspector_url(self):
+ ds = self.congress.list_datasources()['results']
+ doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+ None)
+ congress_endpoint = self.congress.httpclient.get_endpoint(auth=self.auth)
+ return ('%s/v1/data-sources/%s/tables/events/rows' %
+ (congress_endpoint, doctor_ds['id']))
+
+ def start(self):
+ self.log.info('congress inspector start......')
+
+ for rule_name, rule in self.rules.items():
+ self._add_rule(rule_name, rule)
+
+ def stop(self):
+ self.log.info('congress inspector stop......')
+
+ for rule_name in self.rules.keys():
+ self._del_rule(rule_name)
+
+ def _add_rule(self, rule_name, rule):
+ if rule_name not in self.policy_rules:
+ self.congress.create_policy_rule(self.policy,
+ body={'name': rule_name,
+ 'rule': rule})
+
+ def _del_rule(self, rule_name):
+ if rule_name in self.policy_rules:
+ rule_id = self.policy_rules[rule_name]['id']
+ self.congress.delete_policy_rule(self.policy, rule_id)
diff --git a/tests/inspector/sample.py b/tests/inspector/sample.py
new file mode 100644
index 00000000..dda053ab
--- /dev/null
+++ b/tests/inspector/sample.py
@@ -0,0 +1,151 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import collections
+from flask import Flask
+from flask import request
+import json
+import time
+from threading import Thread
+import requests
+
+from identity_auth import get_identity_auth
+from identity_auth import get_session
+from os_clients import nova_client
+from inspector.base import BaseInspector
+
+
+class SampleInspector(BaseInspector):
+ event_type = 'compute.host.down'
+
+ def __init__(self, conf, log):
+ super(SampleInspector, self).__init__(conf, log)
+ self.inspector_url = self.get_inspector_url()
+ self.novaclients = list()
+ self._init_novaclients()
+ # Normally we use this client for non redundant API calls
+ self.nova = self.novaclients[0]
+
+ self.servers = collections.defaultdict(list)
+ self.hostnames = list()
+ self.app = None
+
+ def _init_novaclients(self):
+ self.NUMBER_OF_CLIENTS = self.conf.instance_count
+ auth = get_identity_auth(project=self.conf.doctor_project)
+ session = get_session(auth=auth)
+ for i in range(self.NUMBER_OF_CLIENTS):
+ self.novaclients.append(
+ nova_client(self.conf.nova_version, session))
+
+ def _init_servers_list(self):
+ self.servers.clear()
+ opts = {'all_tenants': True}
+ servers = self.nova.servers.list(search_opts=opts)
+ for server in servers:
+ try:
+ host = server.__dict__.get('OS-EXT-SRV-ATTR:host')
+ self.servers[host].append(server)
+ self.log.debug('get hostname=%s from server=%s' % (host, server))
+ except Exception as e:
+ self.log.info('can not get hostname from server=%s' % server)
+
+ def get_inspector_url(self):
+ return 'http://%s:%s' % (self.conf.inspector.ip, self.conf.inspector.port)
+
+ def start(self):
+ self.log.info('sample inspector start......')
+ self._init_servers_list()
+ self.app = InspectorApp(self.conf.inspector.port, self, self.log)
+ self.app.start()
+
+ def stop(self):
+ self.log.info('sample inspector stop......')
+ if not self.app:
+ return
+ for hostname in self.hostnames:
+ self.nova.services.force_down(hostname, 'nova-compute', False)
+
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ }
+ url = '%s%s' % (self.inspector_url, 'shutdown') \
+ if self.inspector_url.endswith('/') else \
+ '%s%s' % (self.inspector_url, '/shutdown')
+ requests.post(url, data='', headers=headers)
+
+ def handle_events(self, events):
+ for event in events:
+ hostname = event['details']['hostname']
+ event_type = event['type']
+ if event_type == self.event_type:
+ self.hostnames.append(hostname)
+ self.disable_compute_host(hostname)
+
+ def disable_compute_host(self, hostname):
+ threads = []
+ if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS:
+ # TODO(tojuvone): This could be enhanced in future with dynamic
+ # reuse of self.novaclients when all threads in use
+ self.log.error('%d servers in %s. Can handle only %d'%(
+ self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS))
+ for nova, server in zip(self.novaclients, self.servers[hostname]):
+ t = ThreadedResetState(nova, "error", server, self.log)
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+ self.nova.services.force_down(hostname, 'nova-compute', True)
+ self.log.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
+
+
+class ThreadedResetState(Thread):
+
+ def __init__(self, nova, state, server, log):
+ Thread.__init__(self)
+ self.nova = nova
+ self.state = state
+ self.server = server
+ self.log = log
+
+ def run(self):
+ self.nova.servers.reset_state(self.server, self.state)
+ self.log.info('doctor mark vm(%s) error at %s' % (self.server, time.time()))
+
+
+class InspectorApp(Thread):
+
+ def __init__(self, port, inspector, log):
+ Thread.__init__(self)
+ self.port = port
+ self.inspector = inspector
+ self.log = log
+
+ def run(self):
+ app = Flask('inspector')
+
+ @app.route('/events', methods=['PUT'])
+ def event_posted():
+ self.log.info('event posted in sample inspector at %s' % time.time())
+ self.log.info('sample inspector = %s' % self.inspector)
+ self.log.info('sample inspector received data = %s' % request.data)
+ events = json.loads(request.data.decode('utf8'))
+ self.inspector.handle_events(events)
+ return "OK"
+
+ @app.route('/shutdown', methods=['POST'])
+ def shutdown():
+ self.log.info('shutdown inspector app server at %s' % time.time())
+ func = request.environ.get('werkzeug.server.shutdown')
+ if func is None:
+ raise RuntimeError('Not running with the Werkzeug Server')
+ func()
+ return 'inspector app shutting down...'
+
+ app.run(host="0.0.0.0", port=self.port)