path: root/app/monitoring/setup/monitoring_handler.py

###############################################################################
# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
# and others                                                                  #
#                                                                             #
# All rights reserved. This program and the accompanying materials            #
# are made available under the terms of the Apache License, Version 2.0       #
# which accompanies this distribution, and is available at                    #
# http://www.apache.org/licenses/LICENSE-2.0                                  #
###############################################################################
# handle specific setup of monitoring

import os
import json
import subprocess
from socket import *

import copy
import pymongo
import shutil
import stat
import tempfile
from boltons.iterutils import remap

from discover.configuration import Configuration
from discover.fetchers.cli.cli_access import CliAccess
from utils.binary_converter import BinaryConverter
from utils.deep_merge import remerge
from utils.inventory_mgr import InventoryMgr
from utils.logging.full_logger import FullLogger
from utils.mongo_access import MongoAccess
from utils.ssh_conn import SshConn
from utils.ssh_connection import SshConnection, SshError


class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
    PRODUCTION_CONFIG_DIR = '/etc/sensu/conf.d'
    APP_SCRIPTS_FOLDER = 'monitoring/checks'
    REMOTE_SCRIPTS_FOLDER = '/etc/sensu/plugins'

    provision_levels = {
        'none': 0,
        'db': 1,
        'files': 2,
        'deploy': 3
    }

    pending_changes = {}

    fetch_ssl_files = []

    def __init__(self, env):
        super().__init__()
        self.log = FullLogger()
        self.configuration = Configuration()
        self.mechanism_drivers = \
            self.configuration.environment['mechanism_drivers']
        self.env = env
        self.had_errors = False
        self.monitoring_config = self.db.monitoring_config_templates
        try:
            self.env_monitoring_config = self.configuration.get('Monitoring')
        except IndexError:
            self.env_monitoring_config = {}
        self.local_host = self.env_monitoring_config.get('server_ip', '')
        self.scripts_prepared_for_host = {}
        self.replacements = self.env_monitoring_config
        self.inv = InventoryMgr()
        self.config_db = self.db[self.inv.get_coll_name('monitoring_config')]
        self.provision = self.provision_levels['none']
        if self.env_monitoring_config:
            provision = self.env_monitoring_config.get('provision', 'none')
            provision = str.lower(provision)
            self.provision =\
                self.provision_levels.get(provision,
                                          self.provision_levels['none'])

    # create a directory if it does not exist
    @staticmethod
    def make_directory(directory):
        if not os.path.exists(directory):
            os.makedirs(directory)
        return directory

    def get_config_dir(self, sub_dir=''):
        config_folder = self.env_monitoring_config['config_folder'] + \
            (os.sep + sub_dir if sub_dir else '')
        return self.make_directory(config_folder).rstrip(os.sep)

    def prepare_config_file(self, file_type, base_condition):
        condition = base_condition
        condition['type'] = file_type
        sort = [('order', pymongo.ASCENDING)]
        docs = self.monitoring_config.find(condition, sort=sort)
        content = {}
        for doc in docs:
            if not self.check_env_condition(doc):
                return {}
            content.update(doc)
        self.replacements['app_path'] = \
            self.configuration.environment['app_path']
        config = self.content_replace({'config': content.get('config', {})})
        return config

    def check_env_condition(self, doc):
        if 'condition' not in doc:
            return True
        condition = doc['condition']
        if 'mechanism_drivers' not in condition:
            return True
        required_mechanism_drivers = condition['mechanism_drivers']
        if not isinstance(required_mechanism_drivers, list):
            required_mechanism_drivers = [required_mechanism_drivers]
        intersection = [val for val in required_mechanism_drivers
                        if val in self.mechanism_drivers]
        return bool(intersection)

    def content_replace(self, content):
        content_remapped = remap(content, visit=self.fill_values)
        return content_remapped

    def format_string(self, val):
        formatted = val if not isinstance(val, str) or '{' not in val \
            else val.format_map(self.replacements)
        return formatted

    def fill_values(self, path, key, value):
        if not path:
            return key, value
        key_formatted = self.format_string(key)
        value_formatted = self.format_string(value)
        return key_formatted, value_formatted

    def get_config_from_db(self, host, file_type):
        find_tuple = {
            'environment': self.env,
            'host': host,
            'type': file_type
        }
        doc = self.config_db.find_one(find_tuple)
        if not doc:
            return {}
        doc.pop("_id", None)
        return self.decode_mongo_keys(doc)

    def write_config_to_db(self, host, config, file_type):
        find_tuple = {
            'environment': self.env,
            'host': host,
            'type': file_type
        }
        doc = copy.copy(find_tuple)
        doc['config'] = config
        doc = self.encode_mongo_keys(doc)
        if not doc:
            return {}
        self.config_db.update_one(find_tuple, {'$set': doc}, upsert=True)

    def merge_config(self, host, file_type, content):
        """
        merge current monitoring config of host
        with newer content.
        return the merged config
        """
        doc = self.get_config_from_db(host, file_type)
        config = remerge([doc['config'], content.get('config')]) if doc \
            else content.get('config', {})
        self.write_config_to_db(host, config, file_type)
        return config

    def write_config_file(self, file_name, sub_dir, host, content,
                          is_container=False, is_server=False):
        """
        apply environment definitions to the config,
        e.g. replace {server_ip} with the IP or host name for the server
        """
        # save the config to DB first, and while doing that
        # merge it with any existing config on same host
        content = self.merge_config(host, file_name, content)

        if self.provision == self.provision_levels['db']:
            self.log.debug('Monitoring setup kept only in DB')
            return
        # now dump the config to the file
        content_json = json.dumps(content.get('config', content),
                                  sort_keys=True, indent=4)
        content_json += '\n'
        # always write the file locally first
        local_dir = self.make_directory(os.path.join(self.get_config_dir(),
                                        sub_dir.strip(os.path.sep)))
        local_path = os.path.join(local_dir, file_name)
        self.write_to_local_host(local_path, content_json)
        self.track_setup_changes(host, is_container, file_name, local_path,
                                 sub_dir, is_server=is_server)

    def add_changes_for_all_clients(self):
        """
        to debug deployment, add simulated track changes entries.
        no need to add for server, as these are done by server_setup()
        """
        docs = self.config_db.find({'environment': self.env})
        for doc in docs:
            host = doc['host']
            sub_dir = os.path.join('host', host)
            file_name = doc['type']
            config_folder = self.env_monitoring_config['config_folder']
            local_path = os.path.join(config_folder, sub_dir, file_name)
            if host == self.env_monitoring_config['server_ip']:
                continue
            self.track_setup_changes(host, False, file_name, local_path,
                                     sub_dir)

    def get_ssh(self, host, is_container=False, for_sftp=False):
        ssh = SshConnection.get_ssh(host, for_sftp)
        if not ssh:
            conf = self.env_monitoring_config
            if is_container or host == conf['server_ip']:
                host = conf['server_ip']
                port = int(conf['ssh_port'])
                user = conf['ssh_user']
                pwd = conf['ssh_password']
                ssh = SshConnection(host, user, _pwd=pwd, _port=port,
                                    for_sftp=for_sftp)
            else:
                ssh = SshConn(host, for_sftp=for_sftp)
        return ssh

    def track_setup_changes(self, host=None, is_container=False, file_name=None,
                            local_path=None, sub_dir=None,
                            is_server=False,
                            target_mode=None,
                            target_path=PRODUCTION_CONFIG_DIR):
        if host not in self.pending_changes:
            self.pending_changes[host] = {}
        if file_name not in self.pending_changes[host]:
            self.pending_changes[host][file_name] = {
                "host": host,
                "is_container": is_container,
                "is_server": is_server,
                "file_name": file_name,
                "local_path": local_path,
                "sub_dir": sub_dir,
                "target_path": target_path,
                "target_mode": target_mode
            }

    def handle_pending_setup_changes(self):
        if self.provision < self.provision_levels['files']:
            if self.provision == self.provision_levels['db']:
                self.log.info('Monitoring config applied only in DB')
            return True
        self.log.info('applying monitoring setup')
        hosts = {}
        scripts_to_hosts = {}
        for host, host_changes in self.pending_changes.items():
            self.handle_pending_host_setup_changes(host_changes, hosts,
                                                   scripts_to_hosts)
        if self.provision < self.provision_levels['deploy']:
            return True
        if self.fetch_ssl_files:
            self.deploy_ssl_files(list(scripts_to_hosts.keys()))
        for host in scripts_to_hosts.values():
            self.deploy_scripts_to_host(host)
        for host in hosts.values():
            self.deploy_config_to_target(host)
        had_errors = ', with some error(s)' if self.had_errors else ''
        self.log.info('done applying monitoring setup{}'.format(had_errors))
        return not self.had_errors

    def handle_pending_host_setup_changes(self, host_changes, hosts,
                                          scripts_to_hosts):
        if self.provision < self.provision_levels['deploy']:
            self.log.info('Monitoring config not deployed to remote host')
        for file_type, changes in host_changes.items():
            host = changes['host']
            is_container = changes['is_container']
            is_server = changes['is_server']
            local_dir = changes['local_path']
            if local_dir == "scripts":
                scripts_to_hosts[host] = {'host': host, 'is_server': is_server}
                continue
            self.log.debug('applying monitoring setup changes ' +
                           'for host ' + host + ', file type: ' + file_type)
            is_local_host = host == self.local_host
            file_path = os.path.join(self.PRODUCTION_CONFIG_DIR, file_type)
            if not is_server and host not in hosts:
                hosts[host] = {
                    'host': host,
                    'local_dir': local_dir,
                    'is_local_host': is_local_host,
                    'is_container': is_container,
                    'is_server': is_server
                }
            if is_server:
                remote_path = self.PRODUCTION_CONFIG_DIR
                if os.path.isfile(local_dir):
                    remote_path += os.path.sep + os.path.basename(local_dir)
                try:
                    self.write_to_server(local_dir,
                                         remote_path=remote_path,
                                         is_container=is_container)
                except SshError:
                    self.had_errors = True
            elif is_local_host:
                    # write to production configuration directory on local host
                    self.make_directory(self.PRODUCTION_CONFIG_DIR)
                    shutil.copy(changes['local_path'], file_path)
            else:
                # write to remote host prepare dir - use sftp
                if self.provision < self.provision_levels['deploy']:
                    continue
                try:
                    self.write_to_remote_host(host, changes['local_path'])
                except SshError:
                    self.had_errors = True

    def prepare_scripts(self, host, is_server):
        if self.scripts_prepared_for_host.get(host, False):
            return
        gateway_host = SshConn.get_gateway_host(host)
        # copy scripts to host
        scripts_dir = os.path.join(self.env_monitoring_config['app_path'],
                                   self.APP_SCRIPTS_FOLDER)
        script_files = [f for f in os.listdir(scripts_dir)
                        if os.path.isfile(os.path.join(scripts_dir, f))]
        script_mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | \
            stat.S_IROTH | stat.S_IXOTH
        target_host = host if is_server else gateway_host
        self.make_remote_dir(target_host, self.REMOTE_SCRIPTS_FOLDER)
        for file_name in script_files:
            remote_path = os.path.join(self.REMOTE_SCRIPTS_FOLDER, file_name)
            local_path = os.path.join(scripts_dir, file_name)
            if not os.path.isfile(local_path):
                continue
            if is_server:
                ssh = self.get_ssh(target_host, for_sftp=True)
                ssh.copy_file(local_path, remote_path, mode=script_mode)
            else:
                self.copy_to_remote_host(target_host, local_path, remote_path,
                                         mode=script_mode,
                                         make_remote_dir=False)
        self.scripts_prepared_for_host[host] = True

    def deploy_ssl_files(self, hosts: list):
        try:
            monitoring_server = self.env_monitoring_config['server_ip']
            gateway_host = SshConn.get_gateway_host(hosts[0])
            temp_dir = tempfile.TemporaryDirectory()
            for file_path in self.fetch_ssl_files:
                # copy SSL files from the monitoring server
                file_name = os.path.basename(file_path)
                local_path = os.path.join(temp_dir.name, file_name)
                self.get_file(monitoring_server, file_path, local_path)
                #  first copy the files to the gateway
                self.write_to_remote_host(gateway_host, local_path,
                                          remote_path=file_path)
            ssl_path = os.path.commonprefix(self.fetch_ssl_files)
            for host in hosts:
                self.copy_from_gateway_to_host(host, ssl_path, ssl_path)
        except SshError:
            self.had_errors = True

    def deploy_scripts_to_host(self, host_details):
        try:
            host = host_details['host']
            is_server = host_details['is_server']
            self.prepare_scripts(host, is_server)
            remote_path = self.REMOTE_SCRIPTS_FOLDER
            local_path = remote_path + os.path.sep + '*.py'
            if is_server:
                return  # this was done earlier
            self.copy_from_gateway_to_host(host, local_path, remote_path)
        except SshError:
            self.had_errors = True

    def restart_service(self, host: str = None,
                        service: str = 'sensu-client',
                        is_server: bool = False,
                        msg: str =None):
        ssh = self.get_ssh(host)
        cmd = 'sudo service {} restart'.format(service)
        log_msg = msg if msg else 'deploying config to host {}'.format(host)
        self.log.info(log_msg)
        try:
            if is_server:
                ssh.exec(cmd)
            else:
                self.run(cmd, ssh_to_host=host, ssh=ssh)
        except SshError as e:
            if 'Error: Redirecting to /bin/systemctl restart' not in str(e):
                self.had_errors = True

    def deploy_config_to_target(self, host_details):
        try:
            host = host_details['host']
            is_local_host = host_details['is_local_host']
            is_container = host_details['is_container']
            is_server = host_details['is_server']
            local_dir = host_details['local_dir']
            if is_container or is_server or not is_local_host:
                local_dir = os.path.dirname(local_dir)
                if not is_server:
                    self.move_setup_files_to_remote_host(host, local_dir)
                # restart the Sensu client on the remote host,
                # so it takes the new setup
                self.restart_service(host)
        except SshError:
            self.had_errors = True

    def run_cmd_locally(self, cmd):
        try:
            subprocess.popen(cmd.split(),
                             shell=True,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        except subprocess.CalledProcessError as e:
            print("Error running command: " + cmd +
                  ", output: " + self.binary2str(e.output) + "\n")

    def move_setup_files_to_remote_host(self, host, local_dir):
        if self.provision < self.provision_levels['deploy']:
            self.log.info('Monitoring config not written to remote host')
            return
        # need to scp the files from the gateway host to the target host
        remote_path = self.PRODUCTION_CONFIG_DIR
        self.copy_from_gateway_to_host(host, local_dir, remote_path)

    def copy_from_gateway_to_host(self, host, local_dir, remote_path):
        ssh = self.get_ssh(host)
        what_to_copy = local_dir if '*' in local_dir else local_dir + '/*'
        if ssh.is_gateway_host(host):
            # on gateway host, perform a simple copy
            # make sure the source and destination are not the same
            local_dir_base = local_dir[:local_dir.rindex('/*')] \
                if '/*' in local_dir else local_dir
            if local_dir_base.strip('/*') == remote_path.strip('/*'):
                return  # same directory - nothing to do
            cmd = 'cp {} {}'.format(what_to_copy, remote_path)
            self.run(cmd, ssh=ssh)
            return
        self.make_remote_dir(host, remote_path)
        remote_path = ssh.get_user() + '@' + host + ':' + \
            remote_path + os.sep
        self.run_on_gateway('scp {} {}'.format(what_to_copy, remote_path),
                            enable_cache=False,
                            use_sudo=None)

    def make_remote_dir_on_host(self, ssh, host, path, path_is_file=False):
        # make sure we have write permissions in target directories
        dir_path = path
        if path_is_file:
            dir_path = os.path.dirname(dir_path)
        cmd = 'sudo mkdir -p ' + dir_path
        try:
            self.run(cmd, ssh_to_host=host, ssh=ssh)
        except timeout:
            self.log.error('timed out trying to create directory {} on host {}'
                           .format(dir_path, host))
            return
        cmd = 'sudo chown -R ' + ssh.get_user() + ' ' + dir_path
        self.run(cmd, ssh_to_host=host, ssh=ssh)

    def make_remote_dir(self, host, path, path_is_file=False):
        ssh = self.get_ssh(host, for_sftp=True)
        self.make_remote_dir_on_host(ssh, host, path, path_is_file)

    def copy_to_remote_host(self, host, local_path, remote_path, mode=None,
                            make_remote_dir=True):
        # copy the local file to the preparation folder for the remote host
        # on the gateway host
        ssh = self.get_ssh(host)
        gateway_host = ssh.get_gateway_host(host)
        if make_remote_dir:
            self.make_remote_dir(gateway_host, remote_path, path_is_file=True)
        ftp_ssh = self.get_ssh(gateway_host, for_sftp=True)
        ftp_ssh.copy_file(local_path, remote_path, mode)

    def write_to_remote_host(self, host, local_path=None, remote_path=None):
        remote_path = remote_path if remote_path else local_path
        self.copy_to_remote_host(host, local_path, remote_path)

    def write_to_server(self, local_path, remote_path=None, is_container=False):
        host = self.env_monitoring_config['server_ip']
        ssh = self.get_ssh(host, is_container=is_container)
        remote_path = remote_path if remote_path else local_path
        self.make_remote_dir_on_host(ssh, host, remote_path, True)
        # copy to config dir first
        ftp_ssh = self.get_ssh(host, is_container=is_container, for_sftp=True)
        ftp_ssh.copy_file(local_path, remote_path)

    @staticmethod
    def write_to_local_host(file_path, content):
        f = open(file_path, "w")
        f.write(content)
        f.close()
        return file_path

    def get_file(self, host, remote_path, local_path):
        ftp_ssh = self.get_ssh(host, for_sftp=True)
        ftp_ssh.copy_file_from_remote(remote_path, local_path)