diff options
Diffstat (limited to 'doctor_tests')
| -rw-r--r-- | doctor_tests/main.py | 7 | ||||
| -rw-r--r-- | doctor_tests/monitor/collectd.py | 11 | ||||
| -rw-r--r-- | doctor_tests/monitor/collectd_plugin.py | 166 | ||||
| -rw-r--r-- | doctor_tests/scenario/common.py | 6 |
4 files changed, 180 insertions, 10 deletions
diff --git a/doctor_tests/main.py b/doctor_tests/main.py index 006aac9f..de45510b 100644 --- a/doctor_tests/main.py +++ b/doctor_tests/main.py @@ -156,16 +156,17 @@ class DoctorTest(object): self.fault.get_disable_network_log() def run_profiler(self): + test_dir = os.path.split(os.path.realpath(__file__))[0] - log_file = '{0}/{1}'.format(sys.path[0], 'disable_network.log') + log_file = '{0}/{1}'.format(test_dir, 'disable_network.log') reg = '(?<=doctor set link down at )\d+.\d+' linkdown = float(match_rep_in_file(reg, log_file).group(0)) - log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log') + log_file = '{0}/{1}'.format(test_dir, 'doctor.log') reg = '(.* doctor mark vm.* error at )(\d+.\d+)' vmdown = float(match_rep_in_file(reg, log_file).group(2)) - reg = '(?<=doctor mark host.* down at )\d+.\d+' + reg = '(.* doctor mark host.* down at )(\d+.\d+)' hostdown = float(match_rep_in_file(reg, log_file).group(2)) reg = '(?<=doctor monitor detected at )\d+.\d+' diff --git a/doctor_tests/monitor/collectd.py b/doctor_tests/monitor/collectd.py index a22d7edc..4e9329c2 100644 --- a/doctor_tests/monitor/collectd.py +++ b/doctor_tests/monitor/collectd.py @@ -17,7 +17,8 @@ from doctor_tests.monitor.base import BaseMonitor class CollectdMonitor(BaseMonitor): def __init__(self, conf, inspector_url, log): super(CollectdMonitor, self).__init__(conf, inspector_url, log) - self.top_dir = os.path.dirname(sys.path[0]) + monitor_dir = os.path.split(os.path.realpath(__file__))[0] + self.top_dir = os.path.dirname(monitor_dir) tmp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) tmp_sock.connect(("8.8.8.8", 80)) @@ -44,7 +45,7 @@ class CollectdMonitor(BaseMonitor): self.log.info("Collectd monitor start.........") self.compute_host = host.name self.compute_ip = host.ip - f = open("%s/tests/collectd.conf" % self.top_dir, 'w') + f = open("%s/collectd.conf" % self.top_dir, 'w') collectd_conf_file = """ Hostname %s FQDNLookup false @@ -98,7 +99,7 @@ LoadPlugin logfile f.write(collectd_conf_file) f.close() - os.system(" scp %s %s/tests/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip)) + os.system(" scp %s %s/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip)) self.log.info("after first scp") ## @TODO (umar) Always assuming that the interface is assigned an IP if ## interface name is not provided. See if there is a better approach @@ -116,7 +117,7 @@ LoadPlugin logfile fi sudo mv collectd.conf /opt/collectd/etc/collectd.conf\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip, self.interface_name, self.interface_name, self.compute_ip)) self.log.info("after first ssh") - os.system(" scp %s %s/tests/lib/monitors/collectd/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip)) + os.system(" scp %s %s/monitor/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip)) self.log.info("after sec scp") os.system(" ssh %s %s@%s \"sudo pkill collectd; sudo /opt/collectd/sbin/collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip)) self.log.info("after sec ssh") @@ -134,4 +135,4 @@ LoadPlugin logfile sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf sudo rm \"\${collectd_conf}-doctor-saved\" fi\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip)) - os.remove("%s/tests/collectd.conf" % self.top_dir) + os.remove("%s/collectd.conf" % self.top_dir) diff --git a/doctor_tests/monitor/collectd_plugin.py b/doctor_tests/monitor/collectd_plugin.py new file mode 100644 index 00000000..57105f33 --- /dev/null +++ b/doctor_tests/monitor/collectd_plugin.py @@ -0,0 +1,166 @@ +############################################################################## +# Copyright (c) 2017 NEC Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import collectd +import sys +from datetime import datetime +import json +import requests +import time +from requests.exceptions import ConnectionError + +from keystoneauth1 import loading +from keystoneauth1 import session +from congressclient.v1 import client + + +def write_debug(str_write, write_type, compute_user): + file_name = ('/home/%s/monitor.log' % compute_user) + file_tmp = open(file_name, write_type) + file_tmp.write( "%s" % str_write) + file_tmp.close() + + +class DoctorMonitorCollectd(object): + def __init__(self): + self.control_ip = '' + self.compute_user = '' + self.compute_ip = '' + self.host_name = '' + self.inspector_type = '' + self.inspector_url = '' + self.os_auth_url = '' + self.os_username = '' + self.os_password = '' + self.os_project_name = '' + self.os_user_domain_name = '' + self.os_user_domain_id = '' + self.os_project_domain_name = '' + self.os_project_domain_id = '' + self.sess = '' + self.auth = '' + self.inspector_notified = 0 + self.start_notifications = 0 + self.monitor_type = 'sample' + + def config_func(self, config): + for node in config.children: + key = node.key.lower() + val = node.values[0] + + if key == 'compute_host': + self.host_name = val + elif key == 'control_ip': + self.control_ip = val + elif key == 'compute_ip': + self.compute_ip = val + elif key == 'compute_user': + self.compute_user = val + elif key == 'inspector_type': + self.inspector_type = val + elif key == 'os_auth_url': + self.os_auth_url = val + elif key == 'os_username': + self.os_username = val + elif key == 'os_password': + self.os_password = val + elif key == 'os_project_name': + self.os_project_name = val + elif key == 'os_user_domain_name': + self.os_user_domain_name = val + elif key == 'os_user_domain_id': + self.os_user_domain_id = val + elif key == 'os_project_domain_name': + self.os_project_domain_name = val + elif key == 'os_project_domain_id': + self.os_project_domain_id = val + else: + collectd.info('Unknown config key "%s"' % key) + + def init_collectd(self): + write_debug("Compute node collectd monitor start at %s\n\n" % datetime.now().isoformat(), "w", self.compute_user) + + if self.inspector_type == 'sample': + self.inspector_url = ('http://%s:12345/events' % self.control_ip) + elif self.inspector_type == 'congress': + loader = loading.get_plugin_loader('password') + self.auth = loader.load_from_options(auth_url=self.os_auth_url, + username=self.os_username, + password=self.os_password, + project_name=self.os_project_name, + user_domain_name=self.os_user_domain_name, + user_domain_id=self.os_user_domain_id, + project_domain_name=self.os_project_domain_name, + project_domain_id=self.os_project_domain_id) + self.sess=session.Session(auth=self.auth) + congress = client.Client(session=self.sess, service_type='policy') + ds = congress.list_datasources()['results'] + doctor_ds = next((item for item in ds if item['driver'] == 'doctor'), + None) + + congress_endpoint = congress.httpclient.get_endpoint(auth=self.auth) + self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' % + (congress_endpoint, doctor_ds['id'])) + else: + sys.exit() + self.start_notifications = 1 + + + def notify_inspector(self): + event_type = "compute.host.down" + payload = [ + { + 'id': ("monitor_%s_id1" % self.monitor_type), + 'time': datetime.now().isoformat(), + 'type': event_type, + 'details': { + 'hostname': self.host_name, + 'status': 'down', + 'monitor': ("monitor_%s" % self.monitor_type), + 'monitor_event_id': ("monitor_%s_event1" % self.monitor_type) + }, + }, + ] + data = json.dumps(payload) + self.inspector_notified = 1 + + if self.inspector_type == 'sample': + headers = {'content-type': 'application/json'} + try: + requests.post(self.inspector_url, data=data, headers=headers) + except ConnectionError as err: + print err + elif self.inspector_type == 'congress': + # TODO(umar) enhance for token expiry case + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'X-Auth-Token': self.sess.get_token() + } + requests.put(self.inspector_url, data=data, headers=headers) + + + def handle_notif(self, notification, data=None): + if (notification.severity == collectd.NOTIF_FAILURE or + notification.severity == collectd.NOTIF_WARNING): + if (self.start_notifications == 1 and self.inspector_notified == 0): + write_debug("Received down notification: doctor monitor detected at %s\n" % time.time(), "a", self.compute_user) + self.notify_inspector() + + elif notification.severity == collectd.NOTIF_OKAY: + collectd.info("Interface status: UP again %s\n" % time.time()) + else: + collectd.info("Unknown notification severity %s\n" % notification.severity) + + +monitor = DoctorMonitorCollectd() + +collectd.register_config(monitor.config_func) +collectd.register_init(monitor.init_collectd) +collectd.register_notification(monitor.handle_notif) diff --git a/doctor_tests/scenario/common.py b/doctor_tests/scenario/common.py index a5cbe483..9344dcaf 100644 --- a/doctor_tests/scenario/common.py +++ b/doctor_tests/scenario/common.py @@ -6,13 +6,15 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -import sys +import os from doctor_tests.common.utils import match_rep_in_file def calculate_notification_time(): - log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log') + scenario_dir = os.path.split(os.path.realpath(__file__))[0] + doctor_tests_dir = os.path.dirname(scenario_dir) + log_file = '{0}/{1}'.format(doctor_tests_dir, 'doctor.log') reg = '(?<=doctor monitor detected at )\d+.\d+' result = match_rep_in_file(reg, log_file) |