From f6bd0ee9158de10e75011783c087171898705ad0 Mon Sep 17 00:00:00 2001 From: yayogev Date: Wed, 6 Sep 2017 20:17:57 +0300 Subject: US2925 add monitoring of host pNIC in OVS - check_interface.py: renamed to check_pnic_ovs.py - check_interface.py: check changed to use 'ip link show' command - monitor.py: handle check name where object type contains underscore, specifically 'host_pnic' - monitor.py: refactored to work as class - monitoring_pnic.py: changed to handle either OVS or VPP - monitoring_check_handler.py: change to allow defining hard-coded check_type string for file_type naming - monitoring config templates: removed default handler Change-Id: Iad38fa108e9ceae18a7c94b3570a8d9b836a8632 Signed-off-by: yayogev --- app/install/db/monitoring_config_templates.json | 58 ++++--- app/monitoring/checks/check_interface.py | 50 ------ app/monitoring/checks/check_pnic_ovs.py | 45 ++++++ app/monitoring/handlers/handle_host_pnic.py | 31 ++++ app/monitoring/handlers/handle_pnic.py | 29 ---- app/monitoring/handlers/monitor.py | 192 ++++++++++++++--------- app/monitoring/setup/monitoring_check_handler.py | 3 +- app/monitoring/setup/monitoring_pnic.py | 9 +- 8 files changed, 245 insertions(+), 172 deletions(-) delete mode 100755 app/monitoring/checks/check_interface.py create mode 100755 app/monitoring/checks/check_pnic_ovs.py create mode 100644 app/monitoring/handlers/handle_host_pnic.py delete mode 100644 app/monitoring/handlers/handle_pnic.py diff --git a/app/install/db/monitoring_config_templates.json b/app/install/db/monitoring_config_templates.json index 3dbacae..9bddfa2 100644 --- a/app/install/db/monitoring_config_templates.json +++ b/app/install/db/monitoring_config_templates.json @@ -88,8 +88,7 @@ "base" ], "handlers" : [ - "default", - "file", + "file", "osdna-monitor" ] } @@ -150,7 +149,6 @@ }, "keepalive" : { "handlers" : [ - "default", "file" ] } @@ -218,8 +216,7 @@ "base" ], "handlers" : [ - "default", - "file", + "file", "osdna-monitor" ] } @@ -247,8 +244,7 @@ "base" ], "handlers" : [ - "default", - "file", + "file", "osdna-monitor" ] } @@ -270,8 +266,7 @@ "base" ], "handlers" : [ - "default", - "file", + "file", "osdna-monitor" ] } @@ -306,8 +301,7 @@ "base" ], "handlers" : [ - "default", - "file", + "file", "osdna-monitor" ] } @@ -316,34 +310,61 @@ "monitoring_system" : "sensu", "type" : "client_check_link_vnic-vconnector.json" }, -{ +{ "side" : "client", "order" : "1", "condition" : { "mechanism_drivers" : [ - "VPP" + "OVS" ] }, "config" : { "checks" : { "{objtype}_{objid}" : { "interval" : 15, - "command" : "check_pnic_vpp.py", + "command" : "check_pnic_ovs.py {local_name}", "standalone" : true, "type": "metric", "subscribers" : [ "base" ], "handlers" : [ - "default", - "file", + "file", "osdna-monitor" ] } } }, "monitoring_system" : "sensu", - "type" : "client_check_pnic.json" + "type" : "client_check_host_pnic_ovs.json" +}, +{ + "side" : "client", + "order" : "1", + "condition" : { + "mechanism_drivers" : [ + "VPP" + ] + }, + "config" : { + "checks" : { + "{objtype}_{objid}" : { + "interval" : 15, + "command" : "check_pnic_vpp.py", + "standalone" : true, + "type": "metric", + "subscribers" : [ + "base" + ], + "handlers" : [ + "file", + "osdna-monitor" + ] + } + } + }, + "monitoring_system" : "sensu", + "type" : "client_check_host_pnic_vpp.json" }, { "side" : "client", @@ -365,8 +386,7 @@ "base" ], "handlers" : [ - "default", - "file", + "file", "osdna-monitor" ] } diff --git a/app/monitoring/checks/check_interface.py b/app/monitoring/checks/check_interface.py deleted file mode 100755 index 4140dfe..0000000 --- a/app/monitoring/checks/check_interface.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### - -import re -import sys -import subprocess - -from binary_converter import binary2str - - -if len(sys.argv) < 2: - print('name of interface must be specified') - exit(2) -nic_name = str(sys.argv[1]) - -rc = 0 - -try: - out = subprocess.check_output(["ifconfig " + nic_name], - stderr=subprocess.STDOUT, - shell=True) - out = binary2str(out) - lines = out.splitlines() - line_number = 1 - line = -1 - while line_number < len(lines): - line = lines[line_number] - if ' BROADCAST ' in line: - break - line_number += 1 - state_match = re.match('^\W+([A-Z]+)', line) - if not state_match: - rc = 2 - print('Error: failed to find status in ifconfig output: ' + out) - else: - rc = 0 if state_match.group(1) == 'UP' else 2 - print(out) -except subprocess.CalledProcessError as e: - print("Error finding NIC {}: {}\n".format(nic_name, binary2str(e.output))) - rc = 2 - -exit(rc) diff --git a/app/monitoring/checks/check_pnic_ovs.py b/app/monitoring/checks/check_pnic_ovs.py new file mode 100755 index 0000000..c26e42f --- /dev/null +++ b/app/monitoring/checks/check_pnic_ovs.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + +import sys +import subprocess + +from binary_converter import binary2str + + +def nic_not_found(name: str, output: str): + print("Error finding NIC {}{}{}\n".format(name, ': ' if output else '', + output)) + return 2 + +if len(sys.argv) < 2: + print('name of interface must be specified') + exit(2) +nic_name = str(sys.argv[1]) + +rc = 0 + +try: + cmd = 'ip link show | grep -A1 "^[0-9]\+: {}:"'.format(nic_name) + out = subprocess.check_output([cmd], stderr=subprocess.STDOUT, shell=True) + out = binary2str(out) + lines = out.splitlines() + if not lines: + rc = nic_not_found(nic_name, '') + else: + line = lines[0] + if ' state UP ' not in line: + rc = 2 + print(out) +except subprocess.CalledProcessError as e: + rc = nic_not_found(nic_name, binary2str(e.output)) + +exit(rc) diff --git a/app/monitoring/handlers/handle_host_pnic.py b/app/monitoring/handlers/handle_host_pnic.py new file mode 100644 index 0000000..3c5ba87 --- /dev/null +++ b/app/monitoring/handlers/handle_host_pnic.py @@ -0,0 +1,31 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event for pNIC objects + +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler +from utils.special_char_converter import SpecialCharConverter + + +class HandleHostPnic(MonitoringCheckHandler): + + def __init__(self, args): + super().__init__(args) + + def handle(self, obj_id, check_result): + object_id = obj_id[:obj_id.index('-')] + mac = obj_id[obj_id.index('-')+1:] + converter = SpecialCharConverter() + mac_address = converter.decode_special_characters(mac) + object_id += '-' + mac_address + doc = self.doc_by_id(object_id) + if not doc: + return 1 + self.keep_result(doc, check_result) + return check_result['status'] diff --git a/app/monitoring/handlers/handle_pnic.py b/app/monitoring/handlers/handle_pnic.py deleted file mode 100644 index 934bb16..0000000 --- a/app/monitoring/handlers/handle_pnic.py +++ /dev/null @@ -1,29 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -# handle monitoring event for pNIC objects - -from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler - -class HandlePnic(MonitoringCheckHandler): - - def __init__(self, args): - super().__init__(args) - - def handle(self, id, check_result): - object_id = id[:id.index('-')] - mac = id[id.index('-')+1:] - mac_address = '%s:%s:%s:%s:%s:%s' % \ - (mac[0:2], mac[2:4], mac[4:6], mac[6:8], mac[8:10], mac[10:12]) - object_id += '-' + mac_address - doc = self.doc_by_id(object_id) - if not doc: - return 1 - self.keep_result(doc, check_result) - return check_result['status'] diff --git a/app/monitoring/handlers/monitor.py b/app/monitoring/handlers/monitor.py index e147a7d..95ea4aa 100755 --- a/app/monitoring/handlers/monitor.py +++ b/app/monitoring/handlers/monitor.py @@ -15,78 +15,126 @@ import argparse import json import sys +from utils.inventory_mgr import InventoryMgr from utils.mongo_access import MongoAccess from utils.util import ClassResolver -DEFAULTS = { - 'env': 'WebEX-Mirantis@Cisco', - 'inventory': 'inventory', - 'loglevel': 'WARNING' -} - - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("-m", "--mongo_config", nargs="?", type=str, - default="", - help="name of config file with MongoDB server " + - "access details") - parser.add_argument("-e", "--env", nargs="?", type=str, - default=DEFAULTS['env'], - help="name of environment to scan \n" + - "(default: {})".format(DEFAULTS['env'])) - parser.add_argument("-y", "--inventory", nargs="?", type=str, - default=DEFAULTS['inventory'], - help="name of inventory collection \n" + - "(default: {}".format(DEFAULTS['inventory'])) - parser.add_argument('-i', '--inputfile', nargs='?', type=str, - default='', - help="read input from the specifed file \n" + - "(default: from stdin)") - parser.add_argument("-l", "--loglevel", nargs="?", type=str, - default=DEFAULTS["loglevel"], - help="logging level \n(default: '{}')" - .format(DEFAULTS["loglevel"])) - args = parser.parse_args() - return args - -input = None -args = get_args() -MongoAccess.set_config_file(args.mongo_config) -if args.inputfile: - try: - with open(args.inputfile, 'r') as input_file: - input = input_file.read() - except Exception as e: - raise FileNotFoundError("failed to open input file: " + args.inputfile) - exit(1) -else: - input = sys.stdin.read() - if not input: - raise ValueError("No input provided on stdin") - exit(1) - -check_result_full = json.loads(input) -check_client = check_result_full['client'] -check_result = check_result_full['check'] -check_result['id'] = check_result_full['id'] -name = check_result['name'] -status = check_result['status'] -object_type = name[:name.index('_')] -object_id = name[name.index('_')+1:] -if 'environment' in check_client: - args.env = check_client['environment'] - -handler = None -basic_handling_types = ['vedge', 'vservice'] -if object_type in basic_handling_types: - from monitoring.handlers.basic_check_handler import BasicCheckHandler - handler = BasicCheckHandler(args) -else: - module_name = 'handle_' + object_type - handler = ClassResolver.get_instance_single_arg(args, - module_name=module_name, - package_name='monitoring.handlers') -if handler: - handler.handle(object_id, check_result) + +class Monitor: + DEFAULTS = { + 'env': 'WebEX-Mirantis@Cisco', + 'inventory': 'inventory', + 'loglevel': 'WARNING' + } + object_types = [] + + def __init__(self): + self.args = self.get_args() + MongoAccess.set_config_file(self.args.mongo_config) + self.inv = InventoryMgr() + self.inv.set_collections(self.args.inventory) + self.input_text = None + + def get_args(self): + parser = argparse.ArgumentParser() + parser.add_argument("-m", "--mongo_config", nargs="?", type=str, + default="", + help="name of config file with MongoDB server " + + "access details") + parser.add_argument("-e", "--env", nargs="?", type=str, + default=self.DEFAULTS['env'], + help="name of environment to scan \n" + + "(default: {})".format(self.DEFAULTS['env'])) + parser.add_argument("-y", "--inventory", nargs="?", type=str, + default=self.DEFAULTS['inventory'], + help="name of inventory collection \n" + + "(default: {}".format(self.DEFAULTS['inventory'])) + parser.add_argument('-i', '--inputfile', nargs='?', type=str, + default='', + help="read input from the specifed file \n" + + "(default: from stdin)") + parser.add_argument("-l", "--loglevel", nargs="?", type=str, + default=self.DEFAULTS["loglevel"], + help="logging level \n(default: '{}')" + .format(self.DEFAULTS["loglevel"])) + args = parser.parse_args() + return args + + def get_object_types(self) -> list: + if not self.object_types: + docs = self.inv.find_items({'name': 'object_types'}, + collection='constants') + for types_list in docs: + self.object_types = [t['value'] for t in types_list['data']] + if not self.object_types: + raise ValueError('Unable to fetch object types') + return self.object_types + + def match_object_types(self, check_name: str) -> list: + object_types = self.get_object_types() + matches = [t for t in object_types if check_name.startswith(t + '_')] + return matches + + def find_object_type_and_id(self, check_name: str): + # if we have multiple matching host types, then take the longest + # of these. For example, if matches are ['host', 'host_pnic'], + # then take 'host_pnic'. + # To facilitate this, we sort the matches by reverse order. + matching_object_types = sorted(self.match_object_types(check_name), + reverse=True) + if not matching_object_types: + raise ValueError('Unable to match check name "{}" with object type' + .format(check_name)) + obj_type = matching_object_types[0] + obj_id = check_name[len(obj_type)+1:] + return obj_type, obj_id + + def read_input(self): + if self.args.inputfile: + try: + with open(self.args.inputfile, 'r') as input_file: + self.input_text = input_file.read() + except Exception as e: + raise FileNotFoundError("failed to open input file {}: {}" + .format(self.args.inputfile, str(e))) + else: + self.input_text = sys.stdin.read() + if not self.input_text: + raise ValueError("No input provided on stdin") + + def get_handler_by_type(self, obj_type): + module_name = 'handle_' + obj_type + package = 'monitoring.handlers' + handler = ClassResolver.get_instance_single_arg(self.args, + module_name=module_name, + package_name=package) + return handler + + def get_handler(self, obj_type): + basic_handling_types = ['vedge', 'vservice'] + if obj_type not in basic_handling_types: + return self.get_handler_by_type(obj_type) + from monitoring.handlers.basic_check_handler \ + import BasicCheckHandler + return BasicCheckHandler(self.args) + + def process_input(self): + check_result_full = json.loads(self.input_text) + check_client = check_result_full['client'] + check_result = check_result_full['check'] + check_result['id'] = check_result_full['id'] + name = check_result['name'] + object_type, object_id = monitor.find_object_type_and_id(name) + if 'environment' in check_client: + self.args.env = check_client['environment'] + + check_handler = self.get_handler(object_type) + if check_handler: + check_handler.handle(object_id, check_result) + + def process_check_result(self): + self.read_input() + self.process_input() + +monitor = Monitor() +monitor.process_check_result() diff --git a/app/monitoring/setup/monitoring_check_handler.py b/app/monitoring/setup/monitoring_check_handler.py index 1c9a013..c453439 100644 --- a/app/monitoring/setup/monitoring_check_handler.py +++ b/app/monitoring/setup/monitoring_check_handler.py @@ -25,7 +25,8 @@ class MonitoringCheckHandler(MonitoringHandler, SpecialCharConverter): host = self.inv.get_by_id(self.env, o['host']) if host and 'ip_address' in host: self.replacements['client_ip'] = host['ip_address'] - type_str = o['type'] if 'type' in o else 'link_' + o['link_type'] + type_str = values['check_type'] if 'check_type' in values else \ + (o['type'] if 'type' in o else 'link_' + o['link_type']) file_type = 'client_check_' + type_str + '.json' host = o['host'] sub_dir = '/host/' + host diff --git a/app/monitoring/setup/monitoring_pnic.py b/app/monitoring/setup/monitoring_pnic.py index fdc1c94..c1be96f 100644 --- a/app/monitoring/setup/monitoring_pnic.py +++ b/app/monitoring/setup/monitoring_pnic.py @@ -17,4 +17,11 @@ class MonitoringPnic(MonitoringSimpleObject): # add monitoring setup for remote host def create_setup(self, o): - self.setup('host_pnic', o) + type = 'host_pnic' + env_config = self.configuration.get_env_config() + vpp_or_ovs = 'vpp' if 'VPP' in env_config['mechanism_drivers'] \ + else 'ovs' + type_str = '{}_{}'.format(type, vpp_or_ovs) + self.setup(type, o, values={'check_type': type_str, + 'local_name': o['local_name']}) + -- cgit 1.2.3-korg