summaryrefslogtreecommitdiffstats
path: root/doctor_tests
diff options
context:
space:
mode:
Diffstat (limited to 'doctor_tests')
-rw-r--r--doctor_tests/monitor/collectd.py11
-rw-r--r--doctor_tests/monitor/collectd_plugin.py166
2 files changed, 172 insertions, 5 deletions
diff --git a/doctor_tests/monitor/collectd.py b/doctor_tests/monitor/collectd.py
index a22d7edc..4e9329c2 100644
--- a/doctor_tests/monitor/collectd.py
+++ b/doctor_tests/monitor/collectd.py
@@ -17,7 +17,8 @@ from doctor_tests.monitor.base import BaseMonitor
class CollectdMonitor(BaseMonitor):
def __init__(self, conf, inspector_url, log):
super(CollectdMonitor, self).__init__(conf, inspector_url, log)
- self.top_dir = os.path.dirname(sys.path[0])
+ monitor_dir = os.path.split(os.path.realpath(__file__))[0]
+ self.top_dir = os.path.dirname(monitor_dir)
tmp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
tmp_sock.connect(("8.8.8.8", 80))
@@ -44,7 +45,7 @@ class CollectdMonitor(BaseMonitor):
self.log.info("Collectd monitor start.........")
self.compute_host = host.name
self.compute_ip = host.ip
- f = open("%s/tests/collectd.conf" % self.top_dir, 'w')
+ f = open("%s/collectd.conf" % self.top_dir, 'w')
collectd_conf_file = """
Hostname %s
FQDNLookup false
@@ -98,7 +99,7 @@ LoadPlugin logfile
f.write(collectd_conf_file)
f.close()
- os.system(" scp %s %s/tests/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+ os.system(" scp %s %s/collectd.conf %s@%s: " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
self.log.info("after first scp")
## @TODO (umar) Always assuming that the interface is assigned an IP if
## interface name is not provided. See if there is a better approach
@@ -116,7 +117,7 @@ LoadPlugin logfile
fi
sudo mv collectd.conf /opt/collectd/etc/collectd.conf\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip, self.interface_name, self.interface_name, self.compute_ip))
self.log.info("after first ssh")
- os.system(" scp %s %s/tests/lib/monitors/collectd/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
+ os.system(" scp %s %s/monitor/collectd_plugin.py %s@%s:collectd_plugin.py " % (self.ssh_opts_cpu, self.top_dir, self.compute_user, self.compute_ip))
self.log.info("after sec scp")
os.system(" ssh %s %s@%s \"sudo pkill collectd; sudo /opt/collectd/sbin/collectd\" " % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
self.log.info("after sec ssh")
@@ -134,4 +135,4 @@ LoadPlugin logfile
sudo cp -f \"\${collectd_conf}-doctor-saved\" \$collectd_conf
sudo rm \"\${collectd_conf}-doctor-saved\"
fi\" """ % (self.ssh_opts_cpu, self.compute_user, self.compute_ip))
- os.remove("%s/tests/collectd.conf" % self.top_dir)
+ os.remove("%s/collectd.conf" % self.top_dir)
diff --git a/doctor_tests/monitor/collectd_plugin.py b/doctor_tests/monitor/collectd_plugin.py
new file mode 100644
index 00000000..57105f33
--- /dev/null
+++ b/doctor_tests/monitor/collectd_plugin.py
@@ -0,0 +1,166 @@
+##############################################################################
+# Copyright (c) 2017 NEC Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import collectd
+import sys
+from datetime import datetime
+import json
+import requests
+import time
+from requests.exceptions import ConnectionError
+
+from keystoneauth1 import loading
+from keystoneauth1 import session
+from congressclient.v1 import client
+
+
+def write_debug(str_write, write_type, compute_user):
+ file_name = ('/home/%s/monitor.log' % compute_user)
+ file_tmp = open(file_name, write_type)
+ file_tmp.write( "%s" % str_write)
+ file_tmp.close()
+
+
+class DoctorMonitorCollectd(object):
+ def __init__(self):
+ self.control_ip = ''
+ self.compute_user = ''
+ self.compute_ip = ''
+ self.host_name = ''
+ self.inspector_type = ''
+ self.inspector_url = ''
+ self.os_auth_url = ''
+ self.os_username = ''
+ self.os_password = ''
+ self.os_project_name = ''
+ self.os_user_domain_name = ''
+ self.os_user_domain_id = ''
+ self.os_project_domain_name = ''
+ self.os_project_domain_id = ''
+ self.sess = ''
+ self.auth = ''
+ self.inspector_notified = 0
+ self.start_notifications = 0
+ self.monitor_type = 'sample'
+
+ def config_func(self, config):
+ for node in config.children:
+ key = node.key.lower()
+ val = node.values[0]
+
+ if key == 'compute_host':
+ self.host_name = val
+ elif key == 'control_ip':
+ self.control_ip = val
+ elif key == 'compute_ip':
+ self.compute_ip = val
+ elif key == 'compute_user':
+ self.compute_user = val
+ elif key == 'inspector_type':
+ self.inspector_type = val
+ elif key == 'os_auth_url':
+ self.os_auth_url = val
+ elif key == 'os_username':
+ self.os_username = val
+ elif key == 'os_password':
+ self.os_password = val
+ elif key == 'os_project_name':
+ self.os_project_name = val
+ elif key == 'os_user_domain_name':
+ self.os_user_domain_name = val
+ elif key == 'os_user_domain_id':
+ self.os_user_domain_id = val
+ elif key == 'os_project_domain_name':
+ self.os_project_domain_name = val
+ elif key == 'os_project_domain_id':
+ self.os_project_domain_id = val
+ else:
+ collectd.info('Unknown config key "%s"' % key)
+
+ def init_collectd(self):
+ write_debug("Compute node collectd monitor start at %s\n\n" % datetime.now().isoformat(), "w", self.compute_user)
+
+ if self.inspector_type == 'sample':
+ self.inspector_url = ('http://%s:12345/events' % self.control_ip)
+ elif self.inspector_type == 'congress':
+ loader = loading.get_plugin_loader('password')
+ self.auth = loader.load_from_options(auth_url=self.os_auth_url,
+ username=self.os_username,
+ password=self.os_password,
+ project_name=self.os_project_name,
+ user_domain_name=self.os_user_domain_name,
+ user_domain_id=self.os_user_domain_id,
+ project_domain_name=self.os_project_domain_name,
+ project_domain_id=self.os_project_domain_id)
+ self.sess=session.Session(auth=self.auth)
+ congress = client.Client(session=self.sess, service_type='policy')
+ ds = congress.list_datasources()['results']
+ doctor_ds = next((item for item in ds if item['driver'] == 'doctor'),
+ None)
+
+ congress_endpoint = congress.httpclient.get_endpoint(auth=self.auth)
+ self.inspector_url = ('%s/v1/data-sources/%s/tables/events/rows' %
+ (congress_endpoint, doctor_ds['id']))
+ else:
+ sys.exit()
+ self.start_notifications = 1
+
+
+ def notify_inspector(self):
+ event_type = "compute.host.down"
+ payload = [
+ {
+ 'id': ("monitor_%s_id1" % self.monitor_type),
+ 'time': datetime.now().isoformat(),
+ 'type': event_type,
+ 'details': {
+ 'hostname': self.host_name,
+ 'status': 'down',
+ 'monitor': ("monitor_%s" % self.monitor_type),
+ 'monitor_event_id': ("monitor_%s_event1" % self.monitor_type)
+ },
+ },
+ ]
+ data = json.dumps(payload)
+ self.inspector_notified = 1
+
+ if self.inspector_type == 'sample':
+ headers = {'content-type': 'application/json'}
+ try:
+ requests.post(self.inspector_url, data=data, headers=headers)
+ except ConnectionError as err:
+ print err
+ elif self.inspector_type == 'congress':
+ # TODO(umar) enhance for token expiry case
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ 'X-Auth-Token': self.sess.get_token()
+ }
+ requests.put(self.inspector_url, data=data, headers=headers)
+
+
+ def handle_notif(self, notification, data=None):
+ if (notification.severity == collectd.NOTIF_FAILURE or
+ notification.severity == collectd.NOTIF_WARNING):
+ if (self.start_notifications == 1 and self.inspector_notified == 0):
+ write_debug("Received down notification: doctor monitor detected at %s\n" % time.time(), "a", self.compute_user)
+ self.notify_inspector()
+
+ elif notification.severity == collectd.NOTIF_OKAY:
+ collectd.info("Interface status: UP again %s\n" % time.time())
+ else:
+ collectd.info("Unknown notification severity %s\n" % notification.severity)
+
+
+monitor = DoctorMonitorCollectd()
+
+collectd.register_config(monitor.config_func)
+collectd.register_init(monitor.init_collectd)
+collectd.register_notification(monitor.handle_notif)