From 7e83d0876ddb84a45e130eeba28bc40ef53c074b Mon Sep 17 00:00:00 2001 From: Yaron Yogev Date: Thu, 27 Jul 2017 09:02:54 +0300 Subject: Calipso initial release for OPNFV Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88 Signed-off-by: Yaron Yogev --- app/monitoring/__init__.py | 10 + app/monitoring/checks/binary_converter.py | 17 + app/monitoring/checks/check_interface.py | 50 +++ app/monitoring/checks/check_ping.py | 121 +++++ app/monitoring/checks/check_pnic_vpp.py | 53 +++ app/monitoring/checks/check_vedge_ovs.py | 43 ++ app/monitoring/checks/check_vedge_vpp.py | 50 +++ app/monitoring/checks/check_vnic_vconnector.py | 72 +++ app/monitoring/checks/check_vnic_vpp.py | 48 ++ app/monitoring/checks/check_vservice.py | 82 ++++ app/monitoring/handlers/__init__.py | 10 + app/monitoring/handlers/basic_check_handler.py | 25 ++ app/monitoring/handlers/handle_link.py | 36 ++ app/monitoring/handlers/handle_otep.py | 48 ++ app/monitoring/handlers/handle_pnic.py | 29 ++ app/monitoring/handlers/handle_pnic_vpp.py | 28 ++ app/monitoring/handlers/handle_vnic_vpp.py | 28 ++ app/monitoring/handlers/monitor.py | 92 ++++ .../handlers/monitoring_check_handler.py | 94 ++++ app/monitoring/setup/__init__.py | 10 + app/monitoring/setup/monitoring_check_handler.py | 54 +++ app/monitoring/setup/monitoring_handler.py | 485 +++++++++++++++++++++ app/monitoring/setup/monitoring_host.py | 91 ++++ .../setup/monitoring_link_vnic_vconnector.py | 37 ++ app/monitoring/setup/monitoring_otep.py | 34 ++ app/monitoring/setup/monitoring_pnic.py | 21 + app/monitoring/setup/monitoring_setup_manager.py | 84 ++++ app/monitoring/setup/monitoring_simple_object.py | 25 ++ app/monitoring/setup/monitoring_vedge.py | 19 + app/monitoring/setup/monitoring_vnic.py | 20 + app/monitoring/setup/monitoring_vservice.py | 23 + 31 files changed, 1839 insertions(+) create mode 100644 app/monitoring/__init__.py create mode 100644 app/monitoring/checks/binary_converter.py create mode 100755 app/monitoring/checks/check_interface.py create mode 100755 app/monitoring/checks/check_ping.py create mode 100755 app/monitoring/checks/check_pnic_vpp.py create mode 100755 app/monitoring/checks/check_vedge_ovs.py create mode 100755 app/monitoring/checks/check_vedge_vpp.py create mode 100755 app/monitoring/checks/check_vnic_vconnector.py create mode 100755 app/monitoring/checks/check_vnic_vpp.py create mode 100644 app/monitoring/checks/check_vservice.py create mode 100644 app/monitoring/handlers/__init__.py create mode 100644 app/monitoring/handlers/basic_check_handler.py create mode 100644 app/monitoring/handlers/handle_link.py create mode 100644 app/monitoring/handlers/handle_otep.py create mode 100644 app/monitoring/handlers/handle_pnic.py create mode 100644 app/monitoring/handlers/handle_pnic_vpp.py create mode 100644 app/monitoring/handlers/handle_vnic_vpp.py create mode 100755 app/monitoring/handlers/monitor.py create mode 100644 app/monitoring/handlers/monitoring_check_handler.py create mode 100644 app/monitoring/setup/__init__.py create mode 100644 app/monitoring/setup/monitoring_check_handler.py create mode 100644 app/monitoring/setup/monitoring_handler.py create mode 100644 app/monitoring/setup/monitoring_host.py create mode 100644 app/monitoring/setup/monitoring_link_vnic_vconnector.py create mode 100644 app/monitoring/setup/monitoring_otep.py create mode 100644 app/monitoring/setup/monitoring_pnic.py create mode 100644 app/monitoring/setup/monitoring_setup_manager.py create mode 100644 app/monitoring/setup/monitoring_simple_object.py create mode 100644 app/monitoring/setup/monitoring_vedge.py create mode 100644 app/monitoring/setup/monitoring_vnic.py create mode 100644 app/monitoring/setup/monitoring_vservice.py (limited to 'app/monitoring') diff --git a/app/monitoring/__init__.py b/app/monitoring/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/monitoring/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/monitoring/checks/binary_converter.py b/app/monitoring/checks/binary_converter.py new file mode 100644 index 0000000..4da1107 --- /dev/null +++ b/app/monitoring/checks/binary_converter.py @@ -0,0 +1,17 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +def binary2str(txt): + if not isinstance(txt, bytes): + return str(txt) + try: + s = txt.decode("utf-8") + except TypeError: + s = str(txt) + return s diff --git a/app/monitoring/checks/check_interface.py b/app/monitoring/checks/check_interface.py new file mode 100755 index 0000000..4140dfe --- /dev/null +++ b/app/monitoring/checks/check_interface.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + +import re +import sys +import subprocess + +from binary_converter import binary2str + + +if len(sys.argv) < 2: + print('name of interface must be specified') + exit(2) +nic_name = str(sys.argv[1]) + +rc = 0 + +try: + out = subprocess.check_output(["ifconfig " + nic_name], + stderr=subprocess.STDOUT, + shell=True) + out = binary2str(out) + lines = out.splitlines() + line_number = 1 + line = -1 + while line_number < len(lines): + line = lines[line_number] + if ' BROADCAST ' in line: + break + line_number += 1 + state_match = re.match('^\W+([A-Z]+)', line) + if not state_match: + rc = 2 + print('Error: failed to find status in ifconfig output: ' + out) + else: + rc = 0 if state_match.group(1) == 'UP' else 2 + print(out) +except subprocess.CalledProcessError as e: + print("Error finding NIC {}: {}\n".format(nic_name, binary2str(e.output))) + rc = 2 + +exit(rc) diff --git a/app/monitoring/checks/check_ping.py b/app/monitoring/checks/check_ping.py new file mode 100755 index 0000000..35e7234 --- /dev/null +++ b/app/monitoring/checks/check_ping.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + +import argparse +import re +import sys +import subprocess + +from binary_converter import binary2str + + +if len(sys.argv) < 2: + raise ValueError('destination address must be specified') + + +def thresholds_string(string): + matches = re.match('\d+%/\d+([.]\d+)?/\d+([.]\d+)?', string) + if not matches: + msg = "%r is not a valid thresholds string" % string + raise argparse.ArgumentTypeError(msg) + return string + + +def get_args(): + # try to read scan plan from command line parameters + parser = argparse.ArgumentParser() + + parser.add_argument("-W", "--warning", nargs="?", + type=thresholds_string, + default='1%/300/600', + help="warning thresholds: packet-loss " + "(%)/avg-rtt (ms)/max-rtt (ms)" + "(example: 1%/300ms/600ms)") + parser.add_argument("-C", "--critical", nargs="?", + type=thresholds_string, + default='10%/1000/2000', + help="critical thresholds: packet-loss " + "(%)/avg-rtt (ms)/max-rtt (ms)" + "(example: 1%/300ms/600ms)") + parser.add_argument("-f", "--source", nargs="?", type=str, default='', + help="source address") + parser.add_argument("-t", "--target", nargs="?", type=str, default='', + help="target address") + parser.add_argument("-c", "--count", nargs="?", type=int, default=5, + help="how many packets will be sent") + parser.add_argument("-i", "--interval", nargs="?", type=float, default=0.5, + help="seconds between sending each packet") + parser.add_argument("-p", "--pattern", nargs="?", type=str, + default='OS-DNA', help="pattern to pad packet with") + parser.add_argument("-w", "--wait", nargs="?", type=int, default=5, + help="seconds to wait for completion of all responses") + parser.add_argument("-s", "--packetsize", nargs="?", type=int, default=256, + help="size of packet vseconds to wait for completion " + "of all responses") + return parser.parse_args() + +args = get_args() + +if not args.target: + raise ValueError('target address must be specified') + +rc = 0 + +try: + cmd = "ping -c {} -i {} -p {} -w {} -s {} {}{} {}".format( + args.count, args.interval, + args.pattern, args.wait, + args.packetsize, + '-I ' if args.source else '', + args.source, args.target) + out = subprocess.check_output([cmd], + stderr=subprocess.STDOUT, + shell=True) + out = binary2str(out) +except subprocess.CalledProcessError as e: + print("Error doing ping: {}\n".format(binary2str(e.output))) + +# find packet loss data +packet_loss_match = re.search('(\d+)[%] packet loss', out, re.M) +if not packet_loss_match: + out += '\npacket loss data not found' + rc = 2 + +# find rtt avg/max data +rtt_results = None +if rc < 2: + regexp = 'rtt min/avg/max/mdev = [0-9.]+/([0-9.]+)/([0-9.]+)/[0-9.]+ ms' + rtt_results = re.search(regexp, out, re.M) + if not rtt_results: + out += '\nrtt results not found' + rc = 2 +if rc < 2: + packet_loss = int(packet_loss_match.group(1)) + avg_rtt = float(rtt_results.group(1)) + max_rtt = float(rtt_results.group(2)) + thresholds_regexp = r'(\d+)%/(\d+[.0-9]*)/(\d+[.0-9]*)' + warn_threshold_match = re.match(thresholds_regexp, args.warning) + critical_threshold_match = re.match(thresholds_regexp, args.critical) + packet_loss_warn = int(warn_threshold_match.group(1)) + packet_loss_critical = int(critical_threshold_match.group(1)) + avg_rtt_warn = float(warn_threshold_match.group(2)) + avg_rtt_critical = float(critical_threshold_match.group(2)) + max_rtt_warn = float(warn_threshold_match.group(3)) + max_rtt_critical = float(critical_threshold_match.group(3)) + if packet_loss > packet_loss_critical or avg_rtt >= avg_rtt_critical or \ + max_rtt >= max_rtt_critical: + rc = 2 + elif packet_loss > packet_loss_warn or avg_rtt >= avg_rtt_warn or \ + max_rtt >= max_rtt_warn: + rc = 1 + +print(out) +exit(rc) diff --git a/app/monitoring/checks/check_pnic_vpp.py b/app/monitoring/checks/check_pnic_vpp.py new file mode 100755 index 0000000..942fdc2 --- /dev/null +++ b/app/monitoring/checks/check_pnic_vpp.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +""" +sudo vppctl show hardware-interfaces: + +take only the virtual interfaces, e.g. "VirtualEthernet0/0/0" +Status: "OK" if "up" is detected in the interface line, CRITICAL otherwise + +return full text of "vppctl show hardware-interfaces" +""" + +import re +import subprocess + +from binary_converter import binary2str + + +NAME_RE = '^[a-zA-Z]*GigabitEthernet' + +rc = 0 + +try: + out = subprocess.check_output(["sudo vppctl show hardware-interfaces"], + stderr=subprocess.STDOUT, + shell=True) + out = binary2str(out) + lines = out.splitlines() + name_re = re.compile(NAME_RE) + matching_lines = [l for l in lines if name_re.search(l)] + matching_line = matching_lines[0] if matching_lines else None + if matching_line: + rc = 0 if "up" in matching_line.split() else 2 + print('output from "vppctl show hardware-interfaces":\n{}' + .format(out)) + else: + rc = 2 + print('Error: failed to find pNic in output of ' + '"vppctl show hardware-interfaces": {}' + .format(out)) +except subprocess.CalledProcessError as e: + print("Error running 'vppctl show hardware-interfaces': {}" + .format(binary2str(e.output))) + rc = 2 + +exit(rc) diff --git a/app/monitoring/checks/check_vedge_ovs.py b/app/monitoring/checks/check_vedge_ovs.py new file mode 100755 index 0000000..849af66 --- /dev/null +++ b/app/monitoring/checks/check_vedge_ovs.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +""" +Check OVS vEdge health + +Run command: +ps -aux | grep "\(ovs-vswitchd\|ovsdb-server\)" + +OK if for both ovs-vswitchd AND ovsdb-server processes we see '(healthy)' +otherwise CRITICAL + +return full text output of the command +""" + +import subprocess + +from binary_converter import binary2str + + +rc = 0 +cmd = 'ps aux | grep "\(ovs-vswitchd\|ovsdb-server\): monitoring" | ' + \ + 'grep -v grep' + +try: + out = subprocess.check_output([cmd], stderr=subprocess.STDOUT, shell=True) + out = binary2str(out) + lines = out.splitlines() + matching_lines = [l for l in lines if '(healthy)'] + rc = 0 if len(matching_lines) == 2 else 2 + print(out) +except subprocess.CalledProcessError as e: + print("Error finding expected output: {}".format(binary2str(e.output))) + rc = 2 + +exit(rc) diff --git a/app/monitoring/checks/check_vedge_vpp.py b/app/monitoring/checks/check_vedge_vpp.py new file mode 100755 index 0000000..346feae --- /dev/null +++ b/app/monitoring/checks/check_vedge_vpp.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +""" +sudo vppctl show runtime: + +test 1: was the return value not null? +test 2: is startup-config-process = done? +1 and 2 = vedge status ok +1 and not 2 = vedge status warning +not 1 = vedge status critical + +return full text of "vppctl show runtime" +""" + +import re +import subprocess + +from binary_converter import binary2str + + +rc = 0 +search_pattern = re.compile("^startup-config-process ") + +try: + out = subprocess.check_output(["sudo vppctl show runtime"], + stderr=subprocess.STDOUT, + shell=True) + out = binary2str(out) + lines = out.splitlines() + matching_lines = [l for l in lines if search_pattern.match(l)] + matching_line = matching_lines[0] if matching_lines else None + if matching_line and "done" in matching_line.split(): + print(out) + else: + rc = 1 + print('Error: failed to find status in ifconfig output: ' + out) +except subprocess.CalledProcessError as e: + print("Error finding 'vppctl show runtime': {}" + .format(binary2str(e.output))) + rc = 2 + +exit(rc) diff --git a/app/monitoring/checks/check_vnic_vconnector.py b/app/monitoring/checks/check_vnic_vconnector.py new file mode 100755 index 0000000..b0f96cd --- /dev/null +++ b/app/monitoring/checks/check_vnic_vconnector.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + +# find status of vnic-vconnector link +# vconnector object name defines name of bridge +# use "brctl showmacs ", then look for the MAC address + +import re +import sys +import subprocess + +from binary_converter import binary2str + + +if len(sys.argv) < 3: + print('usage: ' + sys.argv[0] + ' ') + exit(2) +bridge_name = str(sys.argv[1]) +mac_address = str(sys.argv[2]) + +rc = 0 + +try: + out = subprocess.check_output(["brctl showmacs " + bridge_name], + stderr=subprocess.STDOUT, + shell=True) + out = binary2str(out) + lines = out.splitlines() + line_number = 1 + line = '' + found = False + while line_number < len(lines): + line = lines[line_number] + if mac_address in line: + found = True + break + line_number += 1 + state_match = re.match('^\W+([A-Z]+)', line) + if not found: + rc = 2 + print('Error: failed to find MAC {}:\n{}\n' + .format(mac_address, out)) + else: + # grab "is local?" and "ageing timer" values + line_parts = line.split() # port, mac address, is local?, ageing timer + is_local = line_parts[2] + ageing_timer = line_parts[3] + msg_format =\ + 'vConnector bridge name: {}\n'\ + 'vNIC MAC address: {}\n'\ + 'is local: {}\n'\ + 'ageing timer: {}\n'\ + 'vNIC MAC address: {}\n'\ + 'command: brctl showmacs {}\n'\ + 'output:\n{}' + msg = msg_format.format(bridge_name, mac_address, is_local, + ageing_timer, mac_address, bridge_name, out) + print(msg) +except subprocess.CalledProcessError as e: + print("Error finding MAC {}: {}\n" + .format(mac_address, binary2str(e.output))) + rc = 2 + +exit(rc) diff --git a/app/monitoring/checks/check_vnic_vpp.py b/app/monitoring/checks/check_vnic_vpp.py new file mode 100755 index 0000000..0f77ddd --- /dev/null +++ b/app/monitoring/checks/check_vnic_vpp.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +""" +sudo vppctl show hardware-interfaces: + +take only the virtual interfaces, e.g. "VirtualEthernet0/0/0" +Status: "OK" if "up" is detected in the interface line, CRITICAL otherwise + +return full text of "vppctl show hardware-interfaces" +""" + +import re +import subprocess + +from binary_converter import binary2str + +rc = 0 +search_pattern = re.compile("^Virtual") + +try: + out = subprocess.check_output(["sudo vppctl show hardware-interfaces"], + stderr=subprocess.STDOUT, + shell=True) + out = binary2str(out) + lines = out.splitlines() + matching_lines = [l for l in lines if search_pattern.match(l)] + matching_line = matching_lines[0] if matching_lines else None + if matching_line and "up" in matching_line.split(): + print('output of "vppctl show hardware-interfaces":\n{}' + .format(out)) + else: + rc = 2 + print('Error: failed to find status in output of ' + '"vppctl show hardware-interfaces": {}'.format(out)) +except subprocess.CalledProcessError as e: + print("Error finding 'vppctl show hardware-interfaces': {}" + .format(binary2str(e.output))) + rc = 2 + +exit(rc) diff --git a/app/monitoring/checks/check_vservice.py b/app/monitoring/checks/check_vservice.py new file mode 100644 index 0000000..a95a46a --- /dev/null +++ b/app/monitoring/checks/check_vservice.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + +""" +for vservice with type T and id X +run on the corresponding host: +ip netns pid X +response is pid(s), for example: +32075 + +For DHCP there are multiple pid, we will take the dnsmasq process + +then run : +ps -uf -p 32075 + +get STAT - "S" and "R" = OK +""" + +import subprocess +import sys + +from binary_converter import binary2str + + +rc = 0 + +args = sys.argv +if len(args) < 3: + print('usage: check_vservice.py ') + exit(2) + +vservice_type = args[1] +vservice_id = args[2] +netns_cmd = 'sudo ip netns pid {}'.format(vservice_id) +pid = '' +ps_cmd = '' +try: + out = subprocess.check_output([netns_cmd], stderr=subprocess.STDOUT, + shell=True) + out = binary2str(out) + lines = out.splitlines() + if not lines: + print('no matching vservice: {}\ncommand: {}\noutput: {}' + .format(vservice_id, netns_cmd, out)) + exit(2) + pid = lines[0] +except subprocess.CalledProcessError as e: + print("Error running '{}': {}" + .format(netns_cmd, binary2str(e.output))) + exit(2) +try: + ps_cmd = 'ps -uf -p {}'.format(pid) + out = subprocess.check_output([ps_cmd], stderr=subprocess.STDOUT, + shell=True) + ps_out = binary2str(out) + lines = ps_out.splitlines() + if not lines: + print('no matching vservice: {}\noutput of {}:\n{}' + .format(vservice_id, netns_cmd, out)) + exit(2) + headers = lines[0].split() + lines = lines[1:] + if vservice_type == 'dhcp' and len(lines) > 1: + lines = [line for line in lines if 'dnsmasq' in line] + values = lines[0].split() + stat_index = headers.index('STAT') + status = values[stat_index] + rc = 0 if status in ['S', 'R'] else 2 + print('{}\n{}\n{}'.format(netns_cmd, ps_cmd, ps_out)) +except subprocess.CalledProcessError as e: + print("Error running '{}': {}".format(ps_cmd, binary2str(e.output))) + rc = 2 + +exit(rc) diff --git a/app/monitoring/handlers/__init__.py b/app/monitoring/handlers/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/monitoring/handlers/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/monitoring/handlers/basic_check_handler.py b/app/monitoring/handlers/basic_check_handler.py new file mode 100644 index 0000000..7c945e8 --- /dev/null +++ b/app/monitoring/handlers/basic_check_handler.py @@ -0,0 +1,25 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event for VPP vEdge objects + +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler + + +class BasicCheckHandler(MonitoringCheckHandler): + + def __init__(self, args): + super().__init__(args) + + def handle(self, id, check_result): + doc = self.doc_by_id(id) + if not doc: + return 1 + self.keep_result(doc, check_result) + return check_result['status'] diff --git a/app/monitoring/handlers/handle_link.py b/app/monitoring/handlers/handle_link.py new file mode 100644 index 0000000..26f4d12 --- /dev/null +++ b/app/monitoring/handlers/handle_link.py @@ -0,0 +1,36 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event for links + +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler + + +class HandleLink(MonitoringCheckHandler): + + def __init__(self, args): + super().__init__(args) + + def handle(self, link_id_from_check, check_result): + # link ID from check is formatted like this: + # __ + link_type = link_id_from_check[:link_id_from_check.index('_')] + remainder = link_id_from_check[len(link_type)+1:] + source_id = remainder[:remainder.index('_')] + target_id = remainder[len(source_id)+1:] + search = { + 'link_type': link_type, + 'source_id': source_id, + 'target_id': target_id + } + doc = self.inv.find_items(search, collection='links', get_single=True) + if not doc: + return 1 + self.keep_result(doc, check_result) + return check_result['status'] diff --git a/app/monitoring/handlers/handle_otep.py b/app/monitoring/handlers/handle_otep.py new file mode 100644 index 0000000..0189625 --- /dev/null +++ b/app/monitoring/handlers/handle_otep.py @@ -0,0 +1,48 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event for OTEP objects + +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler + + +class HandleOtep(MonitoringCheckHandler): + + def __init__(self, args): + super().__init__(args) + + def handle(self, id, check_result): + object_id = id[:id.index('_')] + port_id = id[id.index('_')+1:] + doc = self.doc_by_id(object_id) + if not doc: + return 1 + ports = doc['ports'] + port = ports[port_id] + if not port: + self.log.error('Port not found: ' + port_id) + return 1 + status = check_result['status'] + port['status'] = self.STATUS_LABEL[status] + port['status_value'] = status + port['status_text'] = check_result['output'] + + # set object status based on overall state of ports + status_list = [p['status'] for p in ports.values() if 'status' in p] + # OTEP overall status: + # - Critical if no port is OK + # - Warning if some ports not OK + # - otherwise OK + status = \ + 2 if 'OK' not in status_list \ + else 1 if 'Critical' in status_list or 'Warning' in status_list \ + else 0 + self.set_doc_status(doc, status, None, self.check_ts(check_result)) + self.keep_message(doc, check_result) + return status diff --git a/app/monitoring/handlers/handle_pnic.py b/app/monitoring/handlers/handle_pnic.py new file mode 100644 index 0000000..934bb16 --- /dev/null +++ b/app/monitoring/handlers/handle_pnic.py @@ -0,0 +1,29 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event for pNIC objects + +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler + +class HandlePnic(MonitoringCheckHandler): + + def __init__(self, args): + super().__init__(args) + + def handle(self, id, check_result): + object_id = id[:id.index('-')] + mac = id[id.index('-')+1:] + mac_address = '%s:%s:%s:%s:%s:%s' % \ + (mac[0:2], mac[2:4], mac[4:6], mac[6:8], mac[8:10], mac[10:12]) + object_id += '-' + mac_address + doc = self.doc_by_id(object_id) + if not doc: + return 1 + self.keep_result(doc, check_result) + return check_result['status'] diff --git a/app/monitoring/handlers/handle_pnic_vpp.py b/app/monitoring/handlers/handle_pnic_vpp.py new file mode 100644 index 0000000..47a76e5 --- /dev/null +++ b/app/monitoring/handlers/handle_pnic_vpp.py @@ -0,0 +1,28 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event for VPP vEdge objects + +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler + + +class HandlePnicVpp(MonitoringCheckHandler): + + def __init__(self, args): + super().__init__(args) + + def handle(self, id, check_result): + id = self.decode_special_characters(id) + pnic = self.doc_by_id(id) + if not pnic: + return 1 + self.keep_result(pnic, check_result) + # in vEdge object in corresponding port name, set attributes: + # "status", "status_timestamp", "status_text" + return check_result['status'] diff --git a/app/monitoring/handlers/handle_vnic_vpp.py b/app/monitoring/handlers/handle_vnic_vpp.py new file mode 100644 index 0000000..c7d234d --- /dev/null +++ b/app/monitoring/handlers/handle_vnic_vpp.py @@ -0,0 +1,28 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event for VPP vEdge objects + +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler + + +class HandleVnicVpp(MonitoringCheckHandler): + + def __init__(self, args): + super().__init__(args) + + def handle(self, id, check_result): + is_instance_vnic = id.startswith('instance_vnic') + vnic_type = 'instance_vnic' if is_instance_vnic else 'vservice_vnic' + id = self.decode_special_characters(id[len(vnic_type)+1:]) + doc = self.doc_by_id(id) + if not doc: + return 1 + self.keep_result(doc, check_result) + return check_result['status'] diff --git a/app/monitoring/handlers/monitor.py b/app/monitoring/handlers/monitor.py new file mode 100755 index 0000000..e147a7d --- /dev/null +++ b/app/monitoring/handlers/monitor.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + +# handle monitoring events + +import argparse +import json +import sys + +from utils.mongo_access import MongoAccess +from utils.util import ClassResolver + +DEFAULTS = { + 'env': 'WebEX-Mirantis@Cisco', + 'inventory': 'inventory', + 'loglevel': 'WARNING' +} + + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-m", "--mongo_config", nargs="?", type=str, + default="", + help="name of config file with MongoDB server " + + "access details") + parser.add_argument("-e", "--env", nargs="?", type=str, + default=DEFAULTS['env'], + help="name of environment to scan \n" + + "(default: {})".format(DEFAULTS['env'])) + parser.add_argument("-y", "--inventory", nargs="?", type=str, + default=DEFAULTS['inventory'], + help="name of inventory collection \n" + + "(default: {}".format(DEFAULTS['inventory'])) + parser.add_argument('-i', '--inputfile', nargs='?', type=str, + default='', + help="read input from the specifed file \n" + + "(default: from stdin)") + parser.add_argument("-l", "--loglevel", nargs="?", type=str, + default=DEFAULTS["loglevel"], + help="logging level \n(default: '{}')" + .format(DEFAULTS["loglevel"])) + args = parser.parse_args() + return args + +input = None +args = get_args() +MongoAccess.set_config_file(args.mongo_config) +if args.inputfile: + try: + with open(args.inputfile, 'r') as input_file: + input = input_file.read() + except Exception as e: + raise FileNotFoundError("failed to open input file: " + args.inputfile) + exit(1) +else: + input = sys.stdin.read() + if not input: + raise ValueError("No input provided on stdin") + exit(1) + +check_result_full = json.loads(input) +check_client = check_result_full['client'] +check_result = check_result_full['check'] +check_result['id'] = check_result_full['id'] +name = check_result['name'] +status = check_result['status'] +object_type = name[:name.index('_')] +object_id = name[name.index('_')+1:] +if 'environment' in check_client: + args.env = check_client['environment'] + +handler = None +basic_handling_types = ['vedge', 'vservice'] +if object_type in basic_handling_types: + from monitoring.handlers.basic_check_handler import BasicCheckHandler + handler = BasicCheckHandler(args) +else: + module_name = 'handle_' + object_type + handler = ClassResolver.get_instance_single_arg(args, + module_name=module_name, + package_name='monitoring.handlers') +if handler: + handler.handle(object_id, check_result) diff --git a/app/monitoring/handlers/monitoring_check_handler.py b/app/monitoring/handlers/monitoring_check_handler.py new file mode 100644 index 0000000..51769ab --- /dev/null +++ b/app/monitoring/handlers/monitoring_check_handler.py @@ -0,0 +1,94 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event +import datetime +import sys +from time import gmtime, strftime + +from bson import ObjectId + +from discover.configuration import Configuration +from messages.message import Message +from utils.inventory_mgr import InventoryMgr +from utils.logging.full_logger import FullLogger +from utils.special_char_converter import SpecialCharConverter +from utils.string_utils import stringify_datetime + +TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' +SOURCE_SYSTEM = 'Sensu' +ERROR_LEVEL = ['info', 'warn', 'error'] + + +class MonitoringCheckHandler(SpecialCharConverter): + STATUS_LABEL = ['OK', 'Warning', 'Critical'] + + def __init__(self, args): + super().__init__() + self.log = FullLogger() + self.log.set_loglevel(args.loglevel) + self.env = args.env + try: + self.conf = Configuration(args.mongo_config) + self.inv = InventoryMgr() + self.inv.log.set_loglevel(args.loglevel) + self.inv.set_collections(args.inventory) + except FileNotFoundError: + sys.exit(1) + + def doc_by_id(self, object_id): + doc = self.inv.get_by_id(self.env, object_id) + if not doc: + self.log.warn('No matching object found with ID: ' + object_id) + return doc + + def doc_by_db_id(self, db_id, coll_name=None): + coll = self.inv.collections[coll_name] if coll_name else None + doc = self.inv.find({'_id': ObjectId(db_id)}, + get_single=True, collection=coll) + if not doc: + self.log.warn('No matching object found with DB ID: ' + db_id) + return doc + + def set_doc_status(self, doc, status, status_text, timestamp): + doc['status'] = self.STATUS_LABEL[status] if isinstance(status, int) \ + else status + if status_text: + doc['status_text'] = status_text + doc['status_timestamp'] = strftime(TIME_FORMAT, timestamp) + if 'link_type' in doc: + self.inv.write_link(doc) + else: + self.inv.set(doc) + + @staticmethod + def check_ts(check_result): + return gmtime(check_result['executed']) + + def keep_result(self, doc, check_result): + status = check_result['status'] + ts = self.check_ts(check_result) + self.set_doc_status(doc, status, check_result['output'], ts) + self.keep_message(doc, check_result) + + def keep_message(self, doc, check_result, error_level=None): + msg_id = check_result['id'] + obj_id = doc['id'] + display_context = doc['network_id'] if doc['type'] == 'port'\ + else doc['id'] + level = error_level if error_level\ + else ERROR_LEVEL[check_result['status']] + dt = datetime.datetime.utcfromtimestamp(check_result['executed']) + ts = stringify_datetime(dt) + message = Message(msg_id=msg_id, env=self.env, source=SOURCE_SYSTEM, + object_id=obj_id, object_type=doc['type'], + display_context=display_context, level=level, + msg=check_result, ts=ts) + collection = self.inv.collections['messages'] + collection.insert_one(message.get()) diff --git a/app/monitoring/setup/__init__.py b/app/monitoring/setup/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/monitoring/setup/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/monitoring/setup/monitoring_check_handler.py b/app/monitoring/setup/monitoring_check_handler.py new file mode 100644 index 0000000..1c9a013 --- /dev/null +++ b/app/monitoring/setup/monitoring_check_handler.py @@ -0,0 +1,54 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_handler import MonitoringHandler +from utils.inventory_mgr import InventoryMgr +from utils.special_char_converter import SpecialCharConverter + + +class MonitoringCheckHandler(MonitoringHandler, SpecialCharConverter): + + def __init__(self, env): + super().__init__(env) + + # add monitoring setup on remote host for given object + def create_monitoring_for_object(self, o, values): + self.replacements.update(self.env_monitoring_config) + self.replacements.update(values) + if 'host' in o: + host = self.inv.get_by_id(self.env, o['host']) + if host and 'ip_address' in host: + self.replacements['client_ip'] = host['ip_address'] + type_str = o['type'] if 'type' in o else 'link_' + o['link_type'] + file_type = 'client_check_' + type_str + '.json' + host = o['host'] + sub_dir = '/host/' + host + content = self.prepare_config_file( + file_type, + {'side': 'client', 'type': file_type}) + # need to put this content inside client.json file + client_file = 'client.json' + host = o['host'] + client_file_content = self.get_config_from_db(host, client_file) + # merge checks attribute from current content into client.json + checks = client_file_content['config']['checks'] \ + if (client_file_content and + 'checks' in client_file_content['config']) \ + else {} + checks.update(content.get('config', {}).get('checks', {})) + if client_file_content: + client_file_content['config']['checks'] = checks + else: + client_file_content = { + 'config': { + 'checks': checks + } + } + content = client_file_content + self.write_config_file(client_file, sub_dir, host, content) diff --git a/app/monitoring/setup/monitoring_handler.py b/app/monitoring/setup/monitoring_handler.py new file mode 100644 index 0000000..5b7cae0 --- /dev/null +++ b/app/monitoring/setup/monitoring_handler.py @@ -0,0 +1,485 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle specific setup of monitoring + +import os +import json +import subprocess +from socket import * + +import copy +import pymongo +import shutil +import stat +from boltons.iterutils import remap + +from discover.configuration import Configuration +from discover.fetchers.cli.cli_access import CliAccess +from utils.binary_converter import BinaryConverter +from utils.deep_merge import remerge +from utils.inventory_mgr import InventoryMgr +from utils.logging.full_logger import FullLogger +from utils.mongo_access import MongoAccess +from utils.ssh_conn import SshConn +from utils.ssh_connection import SshConnection + + +class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): + PRODUCTION_CONFIG_DIR = '/etc/sensu/conf.d' + APP_SCRIPTS_FOLDER = 'monitoring/checks' + REMOTE_SCRIPTS_FOLDER = '/etc/sensu/plugins' + TMP_SSL_FOLDER = '/tmp/monitoring_ssl_files' + + provision_levels = { + 'none': 0, + 'db': 1, + 'files': 2, + 'deploy': 3 + } + + pending_changes = {} + + fetch_ssl_files = [] + + def __init__(self, env): + super().__init__() + self.log = FullLogger() + self.configuration = Configuration() + self.mechanism_drivers = \ + self.configuration.environment['mechanism_drivers'] + self.env = env + self.monitoring_config = self.db.monitoring_config_templates + try: + self.env_monitoring_config = self.configuration.get('Monitoring') + except IndexError: + self.env_monitoring_config = {} + self.local_host = self.env_monitoring_config.get('server_ip', '') + self.scripts_prepared_for_host = {} + self.replacements = self.env_monitoring_config + self.inv = InventoryMgr() + self.config_db = self.db[self.inv.get_coll_name('monitoring_config')] + self.provision = self.provision_levels['none'] + if self.env_monitoring_config: + provision = self.env_monitoring_config.get('provision', 'none') + provision = str.lower(provision) + self.provision =\ + self.provision_levels.get(provision, + self.provision_levels['none']) + + # create a directory if it does not exist + @staticmethod + def make_directory(directory): + if not os.path.exists(directory): + os.makedirs(directory) + return directory + + def get_config_dir(self, sub_dir=''): + config_folder = self.env_monitoring_config['config_folder'] + \ + (os.sep + sub_dir if sub_dir else '') + return self.make_directory(config_folder).rstrip(os.sep) + + def prepare_config_file(self, file_type, base_condition): + condition = base_condition + condition['type'] = file_type + sort = [('order', pymongo.ASCENDING)] + docs = self.monitoring_config.find(condition, sort=sort) + content = {} + for doc in docs: + if not self.check_env_condition(doc): + return {} + content.update(doc) + self.replacements['app_path'] = \ + self.configuration.environment['app_path'] + config = self.content_replace({'config': content.get('config', {})}) + return config + + def check_env_condition(self, doc): + if 'condition' not in doc: + return True + condition = doc['condition'] + if 'mechanism_drivers' not in condition: + return True + required_mechanism_drivers = condition['mechanism_drivers'] + if not isinstance(required_mechanism_drivers, list): + required_mechanism_drivers = [required_mechanism_drivers] + intersection = [val for val in required_mechanism_drivers + if val in self.mechanism_drivers] + return bool(intersection) + + def content_replace(self, content): + content_remapped = remap(content, visit=self.fill_values) + return content_remapped + + def format_string(self, val): + formatted = val if not isinstance(val, str) or '{' not in val \ + else val.format_map(self.replacements) + return formatted + + def fill_values(self, path, key, value): + if not path: + return key, value + key_formatted = self.format_string(key) + value_formatted = self.format_string(value) + return key_formatted, value_formatted + + def get_config_from_db(self, host, file_type): + find_tuple = { + 'environment': self.env, + 'host': host, + 'type': file_type + } + doc = self.config_db.find_one(find_tuple) + if not doc: + return {} + doc.pop("_id", None) + return self.decode_mongo_keys(doc) + + def write_config_to_db(self, host, config, file_type): + find_tuple = { + 'environment': self.env, + 'host': host, + 'type': file_type + } + doc = copy.copy(find_tuple) + doc['config'] = config + doc = self.encode_mongo_keys(doc) + if not doc: + return {} + self.config_db.update_one(find_tuple, {'$set': doc}, upsert=True) + + def merge_config(self, host, file_type, content): + """ + merge current monitoring config of host + with newer content. + return the merged config + """ + doc = self.get_config_from_db(host, file_type) + config = remerge([doc['config'], content.get('config')]) if doc \ + else content.get('config', {}) + self.write_config_to_db(host, config, file_type) + return config + + def write_config_file(self, file_name, sub_dir, host, content, + is_container=False, is_server=False): + """ + apply environment definitions to the config, + e.g. replace {server_ip} with the IP or host name for the server + """ + # save the config to DB first, and while doing that + # merge it with any existing config on same host + content = self.merge_config(host, file_name, content) + + if self.provision == self.provision_levels['db']: + self.log.debug('Monitoring setup kept only in DB') + return + # now dump the config to the file + content_json = json.dumps(content.get('config', content), + sort_keys=True, indent=4) + content_json += '\n' + # always write the file locally first + local_dir = self.make_directory(os.path.join(self.get_config_dir(), + sub_dir.strip(os.path.sep))) + local_path = os.path.join(local_dir, file_name) + self.write_to_local_host(local_path, content_json) + self.track_setup_changes(host, is_container, file_name, local_path, + sub_dir, is_server=is_server) + + def add_changes_for_all_clients(self): + """ + to debug deployment, add simulated track changes entries. + no need to add for server, as these are done by server_setup() + """ + docs = self.config_db.find({'environment': self.env}) + for doc in docs: + host = doc['host'] + sub_dir = os.path.join('host', host) + file_name = doc['type'] + config_folder = self.env_monitoring_config['config_folder'] + local_path = os.path.join(config_folder, sub_dir, file_name) + if host == self.env_monitoring_config['server_ip']: + continue + self.track_setup_changes(host, False, file_name, local_path, + sub_dir) + + def get_ssh(self, host, is_container=False, for_sftp=False): + ssh = SshConnection.get_ssh(host, for_sftp) + if not ssh: + if is_container: + conf = self.env_monitoring_config + host = conf['server_ip'] + port = int(conf['ssh_port']) + user = conf['ssh_user'] + pwd = conf['ssh_password'] + ssh = SshConnection(host, user, _pwd=pwd, _port=port, + for_sftp=for_sftp) + else: + ssh = SshConn(host, for_sftp=for_sftp) + return ssh + + def track_setup_changes(self, host=None, is_container=False, file_name=None, + local_path=None, sub_dir=None, + is_server=False, + target_mode=None, + target_path=PRODUCTION_CONFIG_DIR): + if host not in self.pending_changes: + self.pending_changes[host] = {} + if file_name not in self.pending_changes[host]: + self.pending_changes[host][file_name] = { + "host": host, + "is_container": is_container, + "is_server": is_server, + "file_name": file_name, + "local_path": local_path, + "sub_dir": sub_dir, + "target_path": target_path, + "target_mode": target_mode + } + + def handle_pending_setup_changes(self): + if self.provision < self.provision_levels['files']: + if self.provision == self.provision_levels['db']: + self.log.info('Monitoring config applied only in DB') + return + self.log.info('applying monitoring setup') + hosts = {} + scripts_to_hosts = {} + for host, host_changes in self.pending_changes.items(): + self.handle_pending_host_setup_changes(host_changes, hosts, + scripts_to_hosts) + if self.provision < self.provision_levels['deploy']: + return + if self.fetch_ssl_files: + self.deploy_ssl_files(list(scripts_to_hosts.keys())) + for host in scripts_to_hosts.values(): + self.deploy_scripts_to_host(host) + for host in hosts.values(): + self.deploy_config_to_target(host) + self.log.info('done applying monitoring setup') + + def handle_pending_host_setup_changes(self, host_changes, hosts, + scripts_to_hosts): + if self.provision < self.provision_levels['deploy']: + self.log.info('Monitoring config not deployed to remote host') + for file_type, changes in host_changes.items(): + host = changes['host'] + is_container = changes['is_container'] + is_server = changes['is_server'] + local_dir = changes['local_path'] + if local_dir == "scripts": + scripts_to_hosts[host] = {'host': host, 'is_server': is_server} + continue + self.log.debug('applying monitoring setup changes ' + + 'for host ' + host + ', file type: ' + file_type) + is_local_host = host == self.local_host + file_path = os.path.join(self.PRODUCTION_CONFIG_DIR, file_type) + if not is_server and host not in hosts: + hosts[host] = { + 'host': host, + 'local_dir': local_dir, + 'is_local_host': is_local_host, + 'is_container': is_container, + 'is_server': is_server + } + if is_server: + remote_path = self.PRODUCTION_CONFIG_DIR + if os.path.isfile(local_dir): + remote_path += os.path.sep + os.path.basename(local_dir) + self.write_to_server(local_dir, + remote_path=remote_path, + is_container=is_container) + elif is_local_host: + # write to production configuration directory on local host + self.make_directory(self.PRODUCTION_CONFIG_DIR) + shutil.copy(changes['local_path'], file_path) + else: + # write to remote host prepare dir - use sftp + if self.provision < self.provision_levels['deploy']: + continue + self.write_to_remote_host(host, changes['local_path']) + + def prepare_scripts(self, host, is_server): + if self.scripts_prepared_for_host.get(host, False): + return + gateway_host = SshConn.get_gateway_host(host) + # copy scripts to host + scripts_dir = os.path.join(self.env_monitoring_config['app_path'], + self.APP_SCRIPTS_FOLDER) + script_files = [f for f in os.listdir(scripts_dir) + if os.path.isfile(os.path.join(scripts_dir, f))] + script_mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | \ + stat.S_IROTH | stat.S_IXOTH + target_host = host if is_server else gateway_host + self.make_remote_dir(target_host, self.REMOTE_SCRIPTS_FOLDER) + for file_name in script_files: + remote_path = os.path.join(self.REMOTE_SCRIPTS_FOLDER, file_name) + local_path = os.path.join(scripts_dir, file_name) + if not os.path.isfile(local_path): + continue + if is_server: + ssh = self.get_ssh(target_host, for_sftp=True) + ssh.copy_file(local_path, remote_path, mode=script_mode) + else: + self.copy_to_remote_host(target_host, local_path, remote_path, + mode=script_mode, + make_remote_dir=False) + self.scripts_prepared_for_host[host] = True + + def deploy_ssl_files(self, hosts: list): + monitoring_server = self.env_monitoring_config['server_ip'] + gateway_host = SshConn.get_gateway_host(hosts[0]) + self.make_directory(self.TMP_SSL_FOLDER) + for file_path in self.fetch_ssl_files: + # copy SSL files from the monitoring server + file_name = os.path.basename(file_path) + local_path = os.path.join(self.TMP_SSL_FOLDER, file_name) + self.get_file(monitoring_server, file_path, local_path) + # first copy the files to the gateway + self.write_to_remote_host(gateway_host, local_path, + remote_path=file_path) + ssl_path = os.path.commonprefix(self.fetch_ssl_files) + for host in hosts: + self.copy_from_gateway_to_host(host, ssl_path, ssl_path) + # remove files from temporary folder + for file_path in self.fetch_ssl_files: + tmp_path = os.path.join(self.TMP_SSL_FOLDER, + os.path.basename(file_path)) + if os.path.exists(tmp_path): + os.remove(tmp_path) # remove files from temporary folder + + def deploy_scripts_to_host(self, host_details): + host = host_details['host'] + is_server = host_details['is_server'] + self.prepare_scripts(host, is_server) + remote_path = self.REMOTE_SCRIPTS_FOLDER + local_path = remote_path + os.path.sep + '*.py' + if is_server: + return # this was done earlier + self.copy_from_gateway_to_host(host, local_path, remote_path) + + def restart_service(self, host: str = None, + service: str = 'sensu-client', + is_server: bool = False, + msg: str =None): + ssh = self.get_ssh(host) + cmd = 'sudo /etc/init.d/{} restart'.format(service) + log_msg = msg if msg else 'deploying config to host {}'.format(host) + self.log.info(log_msg) + if is_server: + ssh.exec(cmd) + else: + self.run(cmd, ssh_to_host=host, ssh=ssh) + + def deploy_config_to_target(self, host_details): + host = host_details['host'] + is_local_host = host_details['is_local_host'] + is_container = host_details['is_container'] + is_server = host_details['is_server'] + local_dir = host_details['local_dir'] + if is_container or is_server or not is_local_host: + local_dir = os.path.dirname(local_dir) + if not is_server: + self.move_setup_files_to_remote_host(host, local_dir) + # restart the Sensu client on the remote host, + # so it takes the new setup + self.restart_service(host) + + def run_cmd_locally(self, cmd): + try: + subprocess.popen(cmd.split(), + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + print("Error running command: " + cmd + + ", output: " + self.binary2str(e.output) + "\n") + + def move_setup_files_to_remote_host(self, host, local_dir): + if self.provision < self.provision_levels['deploy']: + self.log.info('Monitoring config not written to remote host') + return + # need to scp the files from the gateway host to the target host + remote_path = self.PRODUCTION_CONFIG_DIR + self.copy_from_gateway_to_host(host, local_dir, remote_path) + + def copy_from_gateway_to_host(self, host, local_dir, remote_path): + ssh = self.get_ssh(host) + what_to_copy = local_dir if '*' in local_dir else local_dir + '/*' + if ssh.is_gateway_host(host): + # on gateway host, perform a simple copy + # make sure the source and destination are not the same + local_dir_base = local_dir[:local_dir.rindex('/*')] \ + if '/*' in local_dir else local_dir + if local_dir_base.strip('/*') == remote_path.strip('/*'): + return # same directory - nothing to do + cmd = 'cp {} {}'.format(what_to_copy, remote_path) + self.run(cmd, ssh=ssh) + return + self.make_remote_dir(host, remote_path) + remote_path = ssh.get_user() + '@' + host + ':' + \ + remote_path + os.sep + self.run_on_gateway('scp {} {}'.format(what_to_copy, remote_path), + enable_cache=False, + use_sudo=None) + + def make_remote_dir_on_host(self, ssh, host, path, path_is_file=False): + # make sure we have write permissions in target directories + dir_path = path + if path_is_file: + dir_path = os.path.dirname(dir_path) + cmd = 'sudo mkdir -p ' + dir_path + try: + self.run(cmd, ssh_to_host=host, ssh=ssh) + except timeout: + self.log.error('timed out trying to create directory {} on host {}' + .format(dir_path, host)) + return + cmd = 'sudo chown -R ' + ssh.get_user() + ' ' + dir_path + self.run(cmd, ssh_to_host=host, ssh=ssh) + + def make_remote_dir(self, host, path, path_is_file=False): + ssh = self.get_ssh(host, for_sftp=True) + self.make_remote_dir_on_host(ssh, host, path, path_is_file) + + def copy_to_remote_host(self, host, local_path, remote_path, mode=None, + make_remote_dir=True): + # copy the local file to the preparation folder for the remote host + # on the gateway host + ssh = self.get_ssh(host) + gateway_host = ssh.get_gateway_host(host) + if make_remote_dir: + self.make_remote_dir(gateway_host, remote_path, path_is_file=True) + ftp_ssh = self.get_ssh(gateway_host, for_sftp=True) + ftp_ssh.copy_file(local_path, remote_path, mode) + + def write_to_remote_host(self, host, local_path=None, remote_path=None): + remote_path = remote_path if remote_path else local_path + self.copy_to_remote_host(host, local_path, remote_path) + + def write_to_server(self, local_path, remote_path=None, is_container=False): + host = self.env_monitoring_config['server_ip'] + ssh = self.get_ssh(host, is_container=is_container) + remote_path = remote_path if remote_path else local_path + self.make_remote_dir_on_host(ssh, host, remote_path, True) + # copy to config dir first + ftp_ssh = self.get_ssh(host, is_container=is_container, for_sftp=True) + ftp_ssh.copy_file(local_path, remote_path) + + @staticmethod + def write_to_local_host(file_path, content): + f = open(file_path, "w") + f.write(content) + f.close() + return file_path + + def get_file(self, host, remote_path, local_path): + ftp_ssh = self.get_ssh(host, for_sftp=True) + ftp_ssh.copy_file_from_remote(remote_path, local_path) + diff --git a/app/monitoring/setup/monitoring_host.py b/app/monitoring/setup/monitoring_host.py new file mode 100644 index 0000000..800b5a2 --- /dev/null +++ b/app/monitoring/setup/monitoring_host.py @@ -0,0 +1,91 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import copy +import os +from os.path import join, sep + +from monitoring.setup.monitoring_handler import MonitoringHandler + +RABBITMQ_CONFIG_FILE = 'rabbitmq.json' +RABBITMQ_CONFIG_ATTR = 'rabbitmq' + +RABBITMQ_CERT_FILE_ATTR = 'cert_chain_file' +RABBITMQ_PK_FILE_ATTR = 'private_key_file' +TMP_FILES_DIR = '/tmp' + + +class MonitoringHost(MonitoringHandler): + + def __init__(self, env): + super().__init__(env) + + # add monitoring setup for remote host + def create_setup(self, o): + sensu_host_files = [ + 'transport.json', + 'rabbitmq.json', + 'client.json' + ] + server_ip = self.env_monitoring_config['server_ip'] + host_id = o['host'] + sub_dir = join('/host', host_id) + config = copy.copy(self.env_monitoring_config) + env_name = self.configuration.env_name + client_name = env_name + '-' + o['id'] + client_ip = o['ip_address'] if 'ip_address' in o else o['id'] + self.replacements.update(config) + self.replacements.update({ + 'server_ip': server_ip, + 'client_name': client_name, + 'client_ip': client_ip, + 'env_name': env_name + }) + + # copy configuration files + for file_name in sensu_host_files: + content = self.prepare_config_file(file_name, {'side': 'client'}) + self.get_ssl_files(host_id, file_name, content) + self.write_config_file(file_name, sub_dir, host_id, content) + + if self.provision < self.provision_levels['deploy']: + return + + self.track_setup_changes(host_id, False, "", "scripts", None) + + # mark this environment as prepared + self.configuration.update_env({'monitoring_setup_done': True}) + + def get_ssl_files(self, host, file_type, content): + if self.fetch_ssl_files: + return # already got names of SSL files + if file_type != RABBITMQ_CONFIG_FILE: + return + if not isinstance(content, dict): + self.log.warn('invalid content of {}'.format(RABBITMQ_CONFIG_FILE)) + return + config = content['config'] + if not config: + self.log.warn('invalid content of {}'.format(RABBITMQ_CONFIG_FILE)) + return + if RABBITMQ_CONFIG_ATTR not in config: + self.log.warn('invalid content of {}'.format(RABBITMQ_CONFIG_FILE)) + return + ssl_conf = config.get(RABBITMQ_CONFIG_ATTR).get('ssl') + if not ssl_conf: + return # SSL not used + + for path_attr in [RABBITMQ_CERT_FILE_ATTR, RABBITMQ_PK_FILE_ATTR]: + path = ssl_conf.get(path_attr) + if not path: + self.log.error('missing SSL path {}'.format(path_attr)) + return + # this configuration requires SSL + # keep the path of the files for later use + self.fetch_ssl_files.append(path) diff --git a/app/monitoring/setup/monitoring_link_vnic_vconnector.py b/app/monitoring/setup/monitoring_link_vnic_vconnector.py new file mode 100644 index 0000000..18f04e4 --- /dev/null +++ b/app/monitoring/setup/monitoring_link_vnic_vconnector.py @@ -0,0 +1,37 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_check_handler import MonitoringCheckHandler + + +class MonitoringLinkVnicVconnector(MonitoringCheckHandler): + + def __init__(self, env): + super().__init__(env) + + # add monitoring setup for remote host + def create_setup(self, link): + vnic = self.inv.get_by_id(self.env, link['source_id']) + if not vnic: + self.log.error('could not find vnic for vnic-vconnector link') + return + if 'mac_address' not in vnic: + self.log.error('could not find MAC address in vNIC: ' + vnic['id']) + return + vconnector = self.inv.get_by_id(self.env, link['target_id']) + if not vnic: + self.log.error('could not find vconnector for vnic-vconnector link') + return + values = { + 'linktype': 'vnic-vconnector', + 'fromobjid': self.encode_special_characters(vnic['id']), + 'toobjid': vconnector['id'], + 'bridge': vconnector['object_name'], + 'mac_address': vnic['mac_address']} + self.create_monitoring_for_object(link, values) diff --git a/app/monitoring/setup/monitoring_otep.py b/app/monitoring/setup/monitoring_otep.py new file mode 100644 index 0000000..366bd77 --- /dev/null +++ b/app/monitoring/setup/monitoring_otep.py @@ -0,0 +1,34 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_check_handler import MonitoringCheckHandler + + +class MonitoringOtep(MonitoringCheckHandler): + + def __init__(self, env): + super().__init__(env) + + # add monitoring setup for remote host + def create_setup(self, o): + if o['ports']: + for port in o['ports'].values(): + self.create_monitoring_for_otep_port(o, port) + + def create_monitoring_for_otep_port(self, o, port): + if port['type'] not in ['vxlan', 'gre']: + return # we only handle vxlan and gre + opt = port['options'] + values = { + "objtype": "otep", + "objid": o['id'], + "portid": port['name'], + "otep_src_ip": opt['local_ip'], + "otep_dest_ip": opt['remote_ip']} + self.create_monitoring_for_object(o, values) diff --git a/app/monitoring/setup/monitoring_pnic.py b/app/monitoring/setup/monitoring_pnic.py new file mode 100644 index 0000000..d64c8ff --- /dev/null +++ b/app/monitoring/setup/monitoring_pnic.py @@ -0,0 +1,21 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject + + +class MonitoringPnic(MonitoringSimpleObject): + + def __init__(self, env): + super().__init__(env) + + # add monitoring setup for remote host + def create_setup(self, o): + if o.get("pnic_type") != "switch": + self.setup('pnic', o) diff --git a/app/monitoring/setup/monitoring_setup_manager.py b/app/monitoring/setup/monitoring_setup_manager.py new file mode 100644 index 0000000..d6ada33 --- /dev/null +++ b/app/monitoring/setup/monitoring_setup_manager.py @@ -0,0 +1,84 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle adding of monitoring setup as needed + +from monitoring.setup.monitoring_handler import MonitoringHandler +from monitoring.setup.monitoring_host import MonitoringHost +from monitoring.setup.monitoring_link_vnic_vconnector \ + import MonitoringLinkVnicVconnector +from monitoring.setup.monitoring_pnic import MonitoringPnic +from monitoring.setup.monitoring_otep import MonitoringOtep +from monitoring.setup.monitoring_vedge import MonitoringVedge +from monitoring.setup.monitoring_vnic import MonitoringVnic +from monitoring.setup.monitoring_vservice import MonitoringVservice + + +class MonitoringSetupManager(MonitoringHandler): + + object_handlers = None + + def __init__(self, env): + super().__init__(env) + self.object_handlers = { + "host": MonitoringHost(env), + "otep": MonitoringOtep(env), + "vedge": MonitoringVedge(env), + "pnic": MonitoringPnic(env), + "vnic": MonitoringVnic(env), + "vservice": MonitoringVservice(env), + "vnic-vconnector": MonitoringLinkVnicVconnector(env)} + + # add monitoring setup to Sensu server + def server_setup(self): + if self.provision == self.provision_levels['none']: + self.log.debug('Monitoring config setup skipped') + return + sensu_server_files = [ + 'transport.json', + 'client.json', + 'rabbitmq.json', + 'handlers.json', + 'redis.json', + 'api.json' + ] + conf = self.env_monitoring_config + is_container = bool(conf.get('ssh_user', '')) + server_host = conf['server_ip'] + sub_dir = 'server' + self.replacements.update(conf) + for file_name in sensu_server_files: + content = self.prepare_config_file(file_name, {'side': 'server'}) + self.write_config_file(file_name, sub_dir, server_host, content, + is_container=is_container, is_server=True) + # restart sensu server and Uchiwa services + # so it takes the new setup + self.restart_service(host=server_host, service='sensu-server', + is_server=True, + msg='restart sensu-server on {}' + .format(server_host)) + self.restart_service(host=server_host, service='uchiwa', + is_server=True, + msg='restart uchiwa on {}' + .format(server_host)) + self.configuration.update_env({'monitoring_setup_done': True}) + + # add setup for inventory object + def create_setup(self, o): + if self.provision == self.provision_levels['none']: + self.log.debug('Monitoring config setup skipped') + return + type_attribute = 'type' if 'type' in o else 'link_type' + type_value = o[type_attribute] + object_handler = self.object_handlers.get(type_value) + if object_handler: + object_handler.create_setup(o) + + def simulate_track_changes(self): + self.add_changes_for_all_clients() diff --git a/app/monitoring/setup/monitoring_simple_object.py b/app/monitoring/setup/monitoring_simple_object.py new file mode 100644 index 0000000..a66abe0 --- /dev/null +++ b/app/monitoring/setup/monitoring_simple_object.py @@ -0,0 +1,25 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_check_handler import MonitoringCheckHandler + + +class MonitoringSimpleObject(MonitoringCheckHandler): + + def __init__(self, env): + super().__init__(env) + + # add monitoring setup for remote host + def setup(self, type: str, o: dict, values: dict = None): + if not values: + values = {} + values['objtype'] = type + objid = values.get('objid', o['id']) + values['objid'] = self.encode_special_characters(objid) + self.create_monitoring_for_object(o, values) diff --git a/app/monitoring/setup/monitoring_vedge.py b/app/monitoring/setup/monitoring_vedge.py new file mode 100644 index 0000000..144ee3a --- /dev/null +++ b/app/monitoring/setup/monitoring_vedge.py @@ -0,0 +1,19 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject + + +class MonitoringVedge(MonitoringSimpleObject): + + def __init__(self, env): + super().__init__(env) + + def create_setup(self, o): + self.setup('vedge', o) diff --git a/app/monitoring/setup/monitoring_vnic.py b/app/monitoring/setup/monitoring_vnic.py new file mode 100644 index 0000000..7c229f8 --- /dev/null +++ b/app/monitoring/setup/monitoring_vnic.py @@ -0,0 +1,20 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject + + +class MonitoringVnic(MonitoringSimpleObject): + + def __init__(self, env): + super().__init__(env) + + # add monitoring setup for remote host + def create_setup(self, o): + self.setup('vnic', o, values={'vnictype': o['vnic_type']}) diff --git a/app/monitoring/setup/monitoring_vservice.py b/app/monitoring/setup/monitoring_vservice.py new file mode 100644 index 0000000..8313b9b --- /dev/null +++ b/app/monitoring/setup/monitoring_vservice.py @@ -0,0 +1,23 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject + + +class MonitoringVservice(MonitoringSimpleObject): + + def __init__(self, env): + super().__init__(env) + + def create_setup(self, o): + values = { + 'local_service_id': o['local_service_id'], + 'service_type': o['service_type'] + } + self.setup('vservice', o, values=values) -- cgit 1.2.3-korg