release 1.2 + new tagging

Change-Id: I1e876451ec4a330f458dd57adadb15e39969b225
Signed-off-by: Koren Lev <korenlev@gmail.com>

6 files changed, 275 insertions, 4 deletions

diff --git a/app/monitoring/setup/monitoring_check_handler.py b/app/monitoring/setup/monitoring_check_handler.py
index c453439..d1b863d 100644
--- a/app/monitoring/setup/monitoring_check_handler.py
+++ b/app/monitoring/setup/monitoring_check_handler.py
@@ -8,7 +8,6 @@
 # http://www.apache.org/licenses/LICENSE-2.0                                  #
 ###############################################################################
 from monitoring.setup.monitoring_handler import MonitoringHandler
-from utils.inventory_mgr import InventoryMgr
 from utils.special_char_converter import SpecialCharConverter
 
 
@@ -28,14 +27,13 @@ class MonitoringCheckHandler(MonitoringHandler, SpecialCharConverter):
         type_str = values['check_type'] if 'check_type' in values else \
             (o['type'] if 'type' in o else 'link_' + o['link_type'])
         file_type = 'client_check_' + type_str + '.json'
-        host = o['host']
+        host = values['host'] if 'host' in values else o['host']
         sub_dir = '/host/' + host
         content = self.prepare_config_file(
             file_type,
             {'side': 'client', 'type': file_type})
         # need to put this content inside client.json file
         client_file = 'client.json'
-        host = o['host']
         client_file_content = self.get_config_from_db(host, client_file)
         # merge checks attribute from current content into client.json
         checks = client_file_content['config']['checks'] \
@@ -53,3 +51,14 @@ class MonitoringCheckHandler(MonitoringHandler, SpecialCharConverter):
             }
         content = client_file_content
         self.write_config_file(client_file, sub_dir, host, content)
+
+    def get_check_from_db(self, o, postfix=''):
+        client_config = self.get_config_from_db(o. get('host', ''),
+                                                'client.json')
+        if not client_config:
+            return {}
+        checks = client_config.get('config', {}).get('checks', {})
+        objid = self.encode_special_characters(o.get('id', ''))
+        object_check_id = '{}_{}{}'.format(o.get('type'), objid, postfix)
+        check = checks.get(object_check_id, {})
+        return check
diff --git a/app/monitoring/setup/monitoring_host.py b/app/monitoring/setup/monitoring_host.py
index 9450cf6..0b9f420 100644
--- a/app/monitoring/setup/monitoring_host.py
+++ b/app/monitoring/setup/monitoring_host.py
@@ -12,6 +12,7 @@ import os
 from os.path import join, sep
 
 from monitoring.setup.monitoring_handler import MonitoringHandler
+from monitoring.setup.sensu_client_installer import SensuClientInstaller
 
 RABBITMQ_CONFIG_FILE = 'rabbitmq.json'
 RABBITMQ_CONFIG_ATTR = 'rabbitmq'
@@ -27,13 +28,14 @@ class MonitoringHost(MonitoringHandler):
 
     # add monitoring setup for remote host
     def create_setup(self, o):
+        host_id = o.get('host', '')
+        self.install_sensu_on_host(host_id)
         sensu_host_files = [
             'transport.json',
             'rabbitmq.json',
             'client.json'
         ]
         server_ip = self.env_monitoring_config['server_ip']
-        host_id = o['host']
         sub_dir = join('/host', host_id)
         config = copy.copy(self.env_monitoring_config)
         env_name = self.configuration.env_name
@@ -88,3 +90,10 @@ class MonitoringHost(MonitoringHandler):
             # this configuration requires SSL
             # keep the path of the files for later use
             self.fetch_ssl_files.append(path)
+
+    def install_sensu_on_host(self, host_id):
+        auto_install = self.env_monitoring_config \
+            .get('install_monitoring_client', False)
+        if auto_install:
+            installer = SensuClientInstaller(self.env, host_id)
+            installer.install()
diff --git a/app/monitoring/setup/monitoring_instance.py b/app/monitoring/setup/monitoring_instance.py
new file mode 100644
index 0000000..b376441
--- /dev/null
+++ b/app/monitoring/setup/monitoring_instance.py
@@ -0,0 +1,67 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject
+
+
+class MonitoringInstance(MonitoringSimpleObject):
+
+    def __init__(self, env):
+        super().__init__(env)
+
+    # monitoring setup for instance can only be done after vNIC is found
+    # and network for vNIC is set, so the first call will not do anything
+    def create_setup(self, instance: dict):
+        vnics = self.inv.find_items({
+            'environment': self.get_env(),
+            'type': 'vnic',
+            'vnic_type': 'instance_vnic',
+            'id_path': {'$regex': '^{}/'.format(instance['id_path'])}
+        })
+        for vnic in vnics:
+            self.add_instance_communication_monitoring(instance, vnic)
+
+    # for instance we keep list of instance vNICs and services to use in call
+    # to check_instance_communications.py
+    # add this vNIC to the list with the corresponding
+    def add_instance_communication_monitoring(self, instance: dict, vnic: dict):
+        service = self.get_service_for_vnic(vnic)
+        if not service:
+            return
+        check = self.get_check_from_db(instance)
+        services_and_vnics = check.get('command', '')
+        if services_and_vnics:
+            services_and_vnics = \
+                services_and_vnics[services_and_vnics.index('.py')+4:]
+        services_and_vnics_list = \
+            services_and_vnics.split(';') if services_and_vnics \
+            else []
+        service_and_vnic = '{},{}'.format(service.get('local_service_id', ''),
+                                          vnic.get('id'))
+        if service_and_vnic in services_and_vnics_list:
+            return  # we already have this tuple define
+        services_and_vnics_list.append(service_and_vnic)
+        values = {
+            'objtype': 'instance',
+            'objid': self.encode_special_characters(instance['id']),
+            'host': service['host'],
+            'services_and_vnics': ';'.join(services_and_vnics_list)
+        }
+        self.create_monitoring_for_object(instance, values)
+
+    def get_service_for_vnic(self, vnic: dict) -> dict:
+        services = self.inv.find_items({'environment': self.get_env(),
+                                        'type': 'vservice',
+                                        'network': vnic.get('network', '')})
+        if not services:
+            return {}
+        dhcp = next(s for s in services if s.get('service_type') == 'dhcp')
+        if dhcp:
+            return dhcp  # If we have both DHCP and router, return the DHCP
+        return services[0]  # currently only DHCP and router services
diff --git a/app/monitoring/setup/monitoring_setup_manager.py b/app/monitoring/setup/monitoring_setup_manager.py
index bc4fe01..8b7693a 100644
--- a/app/monitoring/setup/monitoring_setup_manager.py
+++ b/app/monitoring/setup/monitoring_setup_manager.py
@@ -11,12 +11,14 @@
 
 from monitoring.setup.monitoring_handler import MonitoringHandler
 from monitoring.setup.monitoring_host import MonitoringHost
+from monitoring.setup.monitoring_instance import MonitoringInstance
 from monitoring.setup.monitoring_link_vnic_vconnector \
     import MonitoringLinkVnicVconnector
 from monitoring.setup.monitoring_pnic import MonitoringPnic
 from monitoring.setup.monitoring_otep import MonitoringOtep
 from monitoring.setup.monitoring_vedge import MonitoringVedge
 from monitoring.setup.monitoring_vnic import MonitoringVnic
+from monitoring.setup.monitoring_vconnector import MonitoringVconnector
 from monitoring.setup.monitoring_vservice import MonitoringVservice
 
 
@@ -31,7 +33,9 @@ class MonitoringSetupManager(MonitoringHandler):
             "otep": MonitoringOtep(env),
             "vedge": MonitoringVedge(env),
             "host_pnic": MonitoringPnic(env),
+            "instance": MonitoringInstance(env),
             "vnic": MonitoringVnic(env),
+            "vconnector": MonitoringVconnector(env),
             "vservice": MonitoringVservice(env),
             "vnic-vconnector": MonitoringLinkVnicVconnector(env)}
 
diff --git a/app/monitoring/setup/monitoring_vconnector.py b/app/monitoring/setup/monitoring_vconnector.py
new file mode 100644
index 0000000..9ddc6af
--- /dev/null
+++ b/app/monitoring/setup/monitoring_vconnector.py
@@ -0,0 +1,24 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject
+
+
+class MonitoringVconnector(MonitoringSimpleObject):
+
+    # add monitoring setup for remote host
+    def create_setup(self, o):
+        type = 'vconnector'
+        env_config = self.configuration.get_env_config()
+        vpp_or_ovs = 'vpp' if 'VPP' in env_config['mechanism_drivers'] \
+            else 'ovs'
+        type_str = '{}_{}'.format(type, vpp_or_ovs)
+        self.setup(type, o, values={'check_type': type_str,
+                                    'name': o['name']})
+
diff --git a/app/monitoring/setup/sensu_client_installer.py b/app/monitoring/setup/sensu_client_installer.py
new file mode 100644
index 0000000..72a8bbb
--- /dev/null
+++ b/app/monitoring/setup/sensu_client_installer.py
@@ -0,0 +1,158 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import os.path
+from pkg_resources import parse_version
+
+from monitoring.setup.monitoring_handler import MonitoringHandler
+from utils.inventory_mgr import InventoryMgr
+
+
+class SensuClientInstaller(MonitoringHandler):
+
+    UBUNTU = 'ubuntu'
+    CENTOS = 'centos'
+
+    INSTALL_CMD = {
+        UBUNTU: 'dpkg -i {}',
+        CENTOS: 'rpm -i {}'
+    }
+    PERMISSIONS_CMD = {
+        UBUNTU: '',
+        CENTOS: 'usermod -aG wheel sensu'
+    }
+    SUDOERS_FILE = '/etc/sudoers'
+
+    available_downloads = {}
+
+    def __init__(self, env: str, host_id: str):
+        super().__init__(env)
+        self.cli_ssh = self.get_ssh(host_id)
+        self.inv = InventoryMgr()
+        self.host = self.inv.get_by_id(env, host_id)
+        self.server = self.env_monitoring_config.get('server_ip')
+        self.server_cli_ssh = self.get_ssh(self.server)
+        self.ubuntu_dist = None
+        self.required_package = None
+
+    def install(self):
+        pkg_to_install = self.get_pkg_to_install()
+        if not pkg_to_install:
+            return
+        try:
+            self.fetch_package(pkg_to_install)
+            self.install_package(pkg_to_install)
+            self.set_permissions()
+        except SystemError as e:
+            self.log.error('Sensu install on host {} failed: {}'
+                           .format(self.host, str(e)))
+            return
+
+    @staticmethod
+    def get_attr_from_output(output_lines: list, attr: str) -> str:
+        matches = [l for l in output_lines if l.startswith(attr)]
+        if not matches:
+            return ''
+        line = matches[0]
+        return SensuClientInstaller.get_attr_from_output_line(line)
+
+    @staticmethod
+    def get_attr_from_output_line(output_line: str):
+        val = output_line[output_line.index(':')+1:].strip()
+        return val
+
+    INSTALLED = 'Installed: '
+    CANDIDATE = 'Candidate: '
+    SENSU_DIR = '/opt/sensu'
+    SENSU_PKG_DIR = '/etc/sensu/pkg'
+    SENSU_PKG_DIR_LOCAL = '/tmp/sensu_pkg'
+    SENSU_VERSION_FILE = '/opt/sensu/version-manifest.txt'
+
+    def find_available_downloads(self):
+        ls_output = self.server_cli_ssh.exec('ls -R {}'
+                                             .format(self.SENSU_PKG_DIR))
+        ls_lines = ls_output.splitlines()
+        last_target_dir = None
+        for line in ls_lines:
+            if line[-4:] in ['/32:', '/64:']:
+                last_target_dir = line.replace(self.SENSU_PKG_DIR, '')
+                continue
+            elif last_target_dir:
+                target_dir = last_target_dir.strip(os.path.sep).strip(':')
+                self.available_downloads[target_dir] = line
+                last_target_dir = None
+            else:
+                last_target_dir = None
+
+    def find_available_package(self, os_details: dict):
+        if not self.available_downloads:
+            self.find_available_downloads()
+        distribution = os_details['ID']
+        version = os_details['version'].split()[-2].lower()
+        arch = os_details['architecure'][-2:]
+        download_dir = os.path.join(distribution, version, arch)
+        download_file = self.available_downloads.get(download_dir)
+        full_path = '' if not download_file \
+            else os.path.join(self.SENSU_PKG_DIR, download_dir, download_file)
+        return download_file, full_path
+
+    @staticmethod
+    def find_available_version(download_file: str) -> str:
+        ver = download_file.replace('sensu', '').strip('-_')
+        ver = ver[:ver.index('-')]
+        return ver
+
+    def get_pkg_to_install(self) -> str:
+        if self.provision == self.provision_levels['none']:
+            return ''
+        if not self.host:
+            return ''
+        supported_os = [self.UBUNTU, self.CENTOS]
+        distribution = self.host['OS']['ID']
+        if distribution not in [self.UBUNTU, self.CENTOS]:
+            self.log.error('Sensu client auto-install only supported for: {}'
+                           .format(', '.join(supported_os)))
+            return ''
+        cmd = 'if [ -d {} ];  then head -1 {} | sed "s/sensu //"; fi' \
+            .format(self.SENSU_DIR, self.SENSU_VERSION_FILE)
+        installed_version = self.cli_ssh.exec(cmd).strip()
+        os_details = self.host['OS']
+        available_pkg, pkg_path = self.find_available_package(os_details)
+        available_version = self.find_available_version(available_pkg)
+        if parse_version(available_version) <= parse_version(installed_version):
+            return ''
+        return pkg_path
+
+    def get_local_path(self, pkg_to_install: str):
+        return os.path.join(self.SENSU_PKG_DIR_LOCAL,
+                            os.path.basename(pkg_to_install))
+
+    def fetch_package(self, pkg_to_install: str):
+        self.make_directory(self.SENSU_PKG_DIR_LOCAL)
+        self.get_file(self.server, pkg_to_install,
+                      self.get_local_path(pkg_to_install))
+        local_path = self.get_local_path(pkg_to_install)
+        self.copy_to_remote_host(self.host['host'],
+                                 local_path=local_path,
+                                 remote_path=local_path)
+
+    def install_package(self, pkg_to_install):
+        local_path = self.get_local_path(pkg_to_install)
+        install_cmd = self.INSTALL_CMD[self.host['OS']['ID']]
+        self.cli_ssh.exec(install_cmd.format(local_path))
+
+    def set_permissions(self):
+        cmd = self.PERMISSIONS_CMD[self.host['OS']['ID']]
+        if cmd:
+            self.cli_ssh.exec(cmd)
+        # add to sudoers file
+        sudoer_permission = 'sensu        ALL=(ALL)       NOPASSWD: ALL'
+        sudoer_cmd = 'grep --silent -w sensu {} || echo "{}" >> {}'\
+            .format(self.SUDOERS_FILE, sudoer_permission, self.SUDOERS_FILE)
+        self.cli_ssh.exec(sudoer_cmd)