diff options
Diffstat (limited to 'app/monitoring/setup')
| -rw-r--r-- | app/monitoring/setup/monitoring_check_handler.py | 15 | ||||
| -rw-r--r-- | app/monitoring/setup/monitoring_host.py | 11 | ||||
| -rw-r--r-- | app/monitoring/setup/monitoring_instance.py | 67 | ||||
| -rw-r--r-- | app/monitoring/setup/monitoring_setup_manager.py | 4 | ||||
| -rw-r--r-- | app/monitoring/setup/monitoring_vconnector.py | 24 | ||||
| -rw-r--r-- | app/monitoring/setup/sensu_client_installer.py | 158 |
6 files changed, 275 insertions, 4 deletions
diff --git a/app/monitoring/setup/monitoring_check_handler.py b/app/monitoring/setup/monitoring_check_handler.py index c453439..d1b863d 100644 --- a/app/monitoring/setup/monitoring_check_handler.py +++ b/app/monitoring/setup/monitoring_check_handler.py @@ -8,7 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### from monitoring.setup.monitoring_handler import MonitoringHandler -from utils.inventory_mgr import InventoryMgr from utils.special_char_converter import SpecialCharConverter @@ -28,14 +27,13 @@ class MonitoringCheckHandler(MonitoringHandler, SpecialCharConverter): type_str = values['check_type'] if 'check_type' in values else \ (o['type'] if 'type' in o else 'link_' + o['link_type']) file_type = 'client_check_' + type_str + '.json' - host = o['host'] + host = values['host'] if 'host' in values else o['host'] sub_dir = '/host/' + host content = self.prepare_config_file( file_type, {'side': 'client', 'type': file_type}) # need to put this content inside client.json file client_file = 'client.json' - host = o['host'] client_file_content = self.get_config_from_db(host, client_file) # merge checks attribute from current content into client.json checks = client_file_content['config']['checks'] \ @@ -53,3 +51,14 @@ class MonitoringCheckHandler(MonitoringHandler, SpecialCharConverter): } content = client_file_content self.write_config_file(client_file, sub_dir, host, content) + + def get_check_from_db(self, o, postfix=''): + client_config = self.get_config_from_db(o. get('host', ''), + 'client.json') + if not client_config: + return {} + checks = client_config.get('config', {}).get('checks', {}) + objid = self.encode_special_characters(o.get('id', '')) + object_check_id = '{}_{}{}'.format(o.get('type'), objid, postfix) + check = checks.get(object_check_id, {}) + return check diff --git a/app/monitoring/setup/monitoring_host.py b/app/monitoring/setup/monitoring_host.py index 9450cf6..0b9f420 100644 --- a/app/monitoring/setup/monitoring_host.py +++ b/app/monitoring/setup/monitoring_host.py @@ -12,6 +12,7 @@ import os from os.path import join, sep from monitoring.setup.monitoring_handler import MonitoringHandler +from monitoring.setup.sensu_client_installer import SensuClientInstaller RABBITMQ_CONFIG_FILE = 'rabbitmq.json' RABBITMQ_CONFIG_ATTR = 'rabbitmq' @@ -27,13 +28,14 @@ class MonitoringHost(MonitoringHandler): # add monitoring setup for remote host def create_setup(self, o): + host_id = o.get('host', '') + self.install_sensu_on_host(host_id) sensu_host_files = [ 'transport.json', 'rabbitmq.json', 'client.json' ] server_ip = self.env_monitoring_config['server_ip'] - host_id = o['host'] sub_dir = join('/host', host_id) config = copy.copy(self.env_monitoring_config) env_name = self.configuration.env_name @@ -88,3 +90,10 @@ class MonitoringHost(MonitoringHandler): # this configuration requires SSL # keep the path of the files for later use self.fetch_ssl_files.append(path) + + def install_sensu_on_host(self, host_id): + auto_install = self.env_monitoring_config \ + .get('install_monitoring_client', False) + if auto_install: + installer = SensuClientInstaller(self.env, host_id) + installer.install() diff --git a/app/monitoring/setup/monitoring_instance.py b/app/monitoring/setup/monitoring_instance.py new file mode 100644 index 0000000..b376441 --- /dev/null +++ b/app/monitoring/setup/monitoring_instance.py @@ -0,0 +1,67 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject + + +class MonitoringInstance(MonitoringSimpleObject): + + def __init__(self, env): + super().__init__(env) + + # monitoring setup for instance can only be done after vNIC is found + # and network for vNIC is set, so the first call will not do anything + def create_setup(self, instance: dict): + vnics = self.inv.find_items({ + 'environment': self.get_env(), + 'type': 'vnic', + 'vnic_type': 'instance_vnic', + 'id_path': {'$regex': '^{}/'.format(instance['id_path'])} + }) + for vnic in vnics: + self.add_instance_communication_monitoring(instance, vnic) + + # for instance we keep list of instance vNICs and services to use in call + # to check_instance_communications.py + # add this vNIC to the list with the corresponding + def add_instance_communication_monitoring(self, instance: dict, vnic: dict): + service = self.get_service_for_vnic(vnic) + if not service: + return + check = self.get_check_from_db(instance) + services_and_vnics = check.get('command', '') + if services_and_vnics: + services_and_vnics = \ + services_and_vnics[services_and_vnics.index('.py')+4:] + services_and_vnics_list = \ + services_and_vnics.split(';') if services_and_vnics \ + else [] + service_and_vnic = '{},{}'.format(service.get('local_service_id', ''), + vnic.get('id')) + if service_and_vnic in services_and_vnics_list: + return # we already have this tuple define + services_and_vnics_list.append(service_and_vnic) + values = { + 'objtype': 'instance', + 'objid': self.encode_special_characters(instance['id']), + 'host': service['host'], + 'services_and_vnics': ';'.join(services_and_vnics_list) + } + self.create_monitoring_for_object(instance, values) + + def get_service_for_vnic(self, vnic: dict) -> dict: + services = self.inv.find_items({'environment': self.get_env(), + 'type': 'vservice', + 'network': vnic.get('network', '')}) + if not services: + return {} + dhcp = next(s for s in services if s.get('service_type') == 'dhcp') + if dhcp: + return dhcp # If we have both DHCP and router, return the DHCP + return services[0] # currently only DHCP and router services diff --git a/app/monitoring/setup/monitoring_setup_manager.py b/app/monitoring/setup/monitoring_setup_manager.py index bc4fe01..8b7693a 100644 --- a/app/monitoring/setup/monitoring_setup_manager.py +++ b/app/monitoring/setup/monitoring_setup_manager.py @@ -11,12 +11,14 @@ from monitoring.setup.monitoring_handler import MonitoringHandler from monitoring.setup.monitoring_host import MonitoringHost +from monitoring.setup.monitoring_instance import MonitoringInstance from monitoring.setup.monitoring_link_vnic_vconnector \ import MonitoringLinkVnicVconnector from monitoring.setup.monitoring_pnic import MonitoringPnic from monitoring.setup.monitoring_otep import MonitoringOtep from monitoring.setup.monitoring_vedge import MonitoringVedge from monitoring.setup.monitoring_vnic import MonitoringVnic +from monitoring.setup.monitoring_vconnector import MonitoringVconnector from monitoring.setup.monitoring_vservice import MonitoringVservice @@ -31,7 +33,9 @@ class MonitoringSetupManager(MonitoringHandler): "otep": MonitoringOtep(env), "vedge": MonitoringVedge(env), "host_pnic": MonitoringPnic(env), + "instance": MonitoringInstance(env), "vnic": MonitoringVnic(env), + "vconnector": MonitoringVconnector(env), "vservice": MonitoringVservice(env), "vnic-vconnector": MonitoringLinkVnicVconnector(env)} diff --git a/app/monitoring/setup/monitoring_vconnector.py b/app/monitoring/setup/monitoring_vconnector.py new file mode 100644 index 0000000..9ddc6af --- /dev/null +++ b/app/monitoring/setup/monitoring_vconnector.py @@ -0,0 +1,24 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject + + +class MonitoringVconnector(MonitoringSimpleObject): + + # add monitoring setup for remote host + def create_setup(self, o): + type = 'vconnector' + env_config = self.configuration.get_env_config() + vpp_or_ovs = 'vpp' if 'VPP' in env_config['mechanism_drivers'] \ + else 'ovs' + type_str = '{}_{}'.format(type, vpp_or_ovs) + self.setup(type, o, values={'check_type': type_str, + 'name': o['name']}) + diff --git a/app/monitoring/setup/sensu_client_installer.py b/app/monitoring/setup/sensu_client_installer.py new file mode 100644 index 0000000..72a8bbb --- /dev/null +++ b/app/monitoring/setup/sensu_client_installer.py @@ -0,0 +1,158 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import os.path +from pkg_resources import parse_version + +from monitoring.setup.monitoring_handler import MonitoringHandler +from utils.inventory_mgr import InventoryMgr + + +class SensuClientInstaller(MonitoringHandler): + + UBUNTU = 'ubuntu' + CENTOS = 'centos' + + INSTALL_CMD = { + UBUNTU: 'dpkg -i {}', + CENTOS: 'rpm -i {}' + } + PERMISSIONS_CMD = { + UBUNTU: '', + CENTOS: 'usermod -aG wheel sensu' + } + SUDOERS_FILE = '/etc/sudoers' + + available_downloads = {} + + def __init__(self, env: str, host_id: str): + super().__init__(env) + self.cli_ssh = self.get_ssh(host_id) + self.inv = InventoryMgr() + self.host = self.inv.get_by_id(env, host_id) + self.server = self.env_monitoring_config.get('server_ip') + self.server_cli_ssh = self.get_ssh(self.server) + self.ubuntu_dist = None + self.required_package = None + + def install(self): + pkg_to_install = self.get_pkg_to_install() + if not pkg_to_install: + return + try: + self.fetch_package(pkg_to_install) + self.install_package(pkg_to_install) + self.set_permissions() + except SystemError as e: + self.log.error('Sensu install on host {} failed: {}' + .format(self.host, str(e))) + return + + @staticmethod + def get_attr_from_output(output_lines: list, attr: str) -> str: + matches = [l for l in output_lines if l.startswith(attr)] + if not matches: + return '' + line = matches[0] + return SensuClientInstaller.get_attr_from_output_line(line) + + @staticmethod + def get_attr_from_output_line(output_line: str): + val = output_line[output_line.index(':')+1:].strip() + return val + + INSTALLED = 'Installed: ' + CANDIDATE = 'Candidate: ' + SENSU_DIR = '/opt/sensu' + SENSU_PKG_DIR = '/etc/sensu/pkg' + SENSU_PKG_DIR_LOCAL = '/tmp/sensu_pkg' + SENSU_VERSION_FILE = '/opt/sensu/version-manifest.txt' + + def find_available_downloads(self): + ls_output = self.server_cli_ssh.exec('ls -R {}' + .format(self.SENSU_PKG_DIR)) + ls_lines = ls_output.splitlines() + last_target_dir = None + for line in ls_lines: + if line[-4:] in ['/32:', '/64:']: + last_target_dir = line.replace(self.SENSU_PKG_DIR, '') + continue + elif last_target_dir: + target_dir = last_target_dir.strip(os.path.sep).strip(':') + self.available_downloads[target_dir] = line + last_target_dir = None + else: + last_target_dir = None + + def find_available_package(self, os_details: dict): + if not self.available_downloads: + self.find_available_downloads() + distribution = os_details['ID'] + version = os_details['version'].split()[-2].lower() + arch = os_details['architecure'][-2:] + download_dir = os.path.join(distribution, version, arch) + download_file = self.available_downloads.get(download_dir) + full_path = '' if not download_file \ + else os.path.join(self.SENSU_PKG_DIR, download_dir, download_file) + return download_file, full_path + + @staticmethod + def find_available_version(download_file: str) -> str: + ver = download_file.replace('sensu', '').strip('-_') + ver = ver[:ver.index('-')] + return ver + + def get_pkg_to_install(self) -> str: + if self.provision == self.provision_levels['none']: + return '' + if not self.host: + return '' + supported_os = [self.UBUNTU, self.CENTOS] + distribution = self.host['OS']['ID'] + if distribution not in [self.UBUNTU, self.CENTOS]: + self.log.error('Sensu client auto-install only supported for: {}' + .format(', '.join(supported_os))) + return '' + cmd = 'if [ -d {} ]; then head -1 {} | sed "s/sensu //"; fi' \ + .format(self.SENSU_DIR, self.SENSU_VERSION_FILE) + installed_version = self.cli_ssh.exec(cmd).strip() + os_details = self.host['OS'] + available_pkg, pkg_path = self.find_available_package(os_details) + available_version = self.find_available_version(available_pkg) + if parse_version(available_version) <= parse_version(installed_version): + return '' + return pkg_path + + def get_local_path(self, pkg_to_install: str): + return os.path.join(self.SENSU_PKG_DIR_LOCAL, + os.path.basename(pkg_to_install)) + + def fetch_package(self, pkg_to_install: str): + self.make_directory(self.SENSU_PKG_DIR_LOCAL) + self.get_file(self.server, pkg_to_install, + self.get_local_path(pkg_to_install)) + local_path = self.get_local_path(pkg_to_install) + self.copy_to_remote_host(self.host['host'], + local_path=local_path, + remote_path=local_path) + + def install_package(self, pkg_to_install): + local_path = self.get_local_path(pkg_to_install) + install_cmd = self.INSTALL_CMD[self.host['OS']['ID']] + self.cli_ssh.exec(install_cmd.format(local_path)) + + def set_permissions(self): + cmd = self.PERMISSIONS_CMD[self.host['OS']['ID']] + if cmd: + self.cli_ssh.exec(cmd) + # add to sudoers file + sudoer_permission = 'sensu ALL=(ALL) NOPASSWD: ALL' + sudoer_cmd = 'grep --silent -w sensu {} || echo "{}" >> {}'\ + .format(self.SUDOERS_FILE, sudoer_permission, self.SUDOERS_FILE) + self.cli_ssh.exec(sudoer_cmd) |