From 692489cc50c8025ede1646627a7a583a4feb3798 Mon Sep 17 00:00:00 2001 From: yayogev Date: Thu, 31 Aug 2017 16:45:23 +0300 Subject: US2876 handle SSH errors ido not stop, but report as 'completed with errors' if there were any errors in SSH calls Change-Id: Ice7e6c4324686adc2d9eec27a9b6187f0fe6808f Signed-off-by: yayogev --- app/discover/scan.py | 6 +- app/discover/scan_manager.py | 20 +++-- app/discover/scanner.py | 16 +++- app/install/db/constants.json | 8 +- app/monitoring/setup/monitoring_handler.py | 113 ++++++++++++++---------- app/test/api/responders_test/test_data/base.py | 1 + app/utils/constants.py | 1 + app/utils/ssh_connection.py | 32 ++++--- ui/imports/api/constants/data/scans-statuses.js | 3 + 9 files changed, 127 insertions(+), 73 deletions(-) diff --git a/app/discover/scan.py b/app/discover/scan.py index 72184ec..86ee990 100755 --- a/app/discover/scan.py +++ b/app/discover/scan.py @@ -274,6 +274,7 @@ class ScanController(Fetcher): # generate ScanObject Class and instance. scanner = Scanner() scanner.set_env(env_name) + scanner.found_errors[env_name] = False # decide what scanning operations to do inventory_only = scan_plan.inventory_only @@ -313,7 +314,10 @@ class ScanController(Fetcher): except ScanError as e: return False, "scan error: " + str(e) SshConnection.disconnect_all() - return True, 'ok' + status = 'ok' if not scanner.found_errors.get(env_name, False) \ + else 'errors detected' + self.log.info('Scan completed, status: {}'.format(status)) + return True, status if __name__ == '__main__': diff --git a/app/discover/scan_manager.py b/app/discover/scan_manager.py index b6ad782..12dbec0 100644 --- a/app/discover/scan_manager.py +++ b/app/discover/scan_manager.py @@ -41,6 +41,8 @@ class ScanManager(Manager): mongo_config_file=self.args.mongo_config) self.db_client = None self.environments_collection = None + self.scans_collection = None + self.scheduled_scans_collection = None @staticmethod def get_args(): @@ -138,8 +140,10 @@ class ScanManager(Manager): def _fail_scan(self, scan_request: dict): self._finalize_scan(scan_request, ScanStatus.FAILED, False) - def _complete_scan(self, scan_request: dict): - self._finalize_scan(scan_request, ScanStatus.COMPLETED, True) + def _complete_scan(self, scan_request: dict, result_message: str): + status = ScanStatus.COMPLETED if result_message == 'ok' \ + else ScanStatus.COMPLETED_WITH_ERRORS + self._finalize_scan(scan_request, status, True) # PyCharm type checker can't reliably check types of document # noinspection PyTypeChecker @@ -184,6 +188,7 @@ class ScanManager(Manager): 'scan_only_links': scheduled_scan['scan_only_links'], 'scan_only_cliques': scheduled_scan['scan_only_cliques'], 'submit_timestamp': ts, + 'interval': interval, 'environment': scheduled_scan['environment'], 'inventory': 'inventory' } @@ -240,8 +245,9 @@ class ScanManager(Manager): time.sleep(self.interval) else: scan_request = results[0] - if not self.inv.is_feature_supported(scan_request.get('environment'), - EnvironmentFeatures.SCANNING): + env = scan_request.get('environment') + scan_feature = EnvironmentFeatures.SCANNING + if not self.inv.is_feature_supported(env, scan_feature): self.log.error("Scanning is not supported for env '{}'" .format(scan_request.get('environment'))) self._fail_scan(scan_request) @@ -281,11 +287,11 @@ class ScanManager(Manager): continue # update the status and timestamps. - self.log.info("Request '{}' has been scanned." - .format(scan_request['_id'])) + self.log.info("Request '{}' has been scanned. ({})" + .format(scan_request['_id'], message)) end_time = datetime.datetime.utcnow() scan_request['end_timestamp'] = end_time - self._complete_scan(scan_request) + self._complete_scan(scan_request, message) finally: self._clean_up() diff --git a/app/discover/scanner.py b/app/discover/scanner.py index 1b7cd51..c310ae7 100644 --- a/app/discover/scanner.py +++ b/app/discover/scanner.py @@ -25,9 +25,8 @@ from discover.find_links_for_vedges import FindLinksForVedges from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics from discover.scan_error import ScanError from discover.scan_metadata_parser import ScanMetadataParser -from utils.constants import EnvironmentFeatures from utils.inventory_mgr import InventoryMgr -from utils.util import ClassResolver +from utils.ssh_connection import SshError class Scanner(Fetcher): @@ -38,6 +37,9 @@ class Scanner(Fetcher): scan_queue = queue.Queue() scan_queue_track = {} + # keep errors indication per environment + found_errors = {} + def __init__(self): """ Scanner is the base class for scanners. @@ -71,6 +73,9 @@ class Scanner(Fetcher): "children": children}) except ValueError: return False + except SshError: + # mark the error + self.found_errors[self.get_env()] = True if limit_to_child_id and len(types_children) > 0: t = types_children[0] children = t["children"] @@ -135,6 +140,9 @@ class Scanner(Fetcher): # It depends on the Fetcher's config. try: db_results = fetcher.get(escaped_id) + except SshError: + self.found_errors[self.get_env()] = True + return [] except Exception as e: self.log.error("Error while scanning : " + "fetcher=%s, " + @@ -233,7 +241,9 @@ class Scanner(Fetcher): clique_scanner.find_cliques() def deploy_monitoring_setup(self): - self.inv.monitoring_setup_manager.handle_pending_setup_changes() + ret = self.inv.monitoring_setup_manager.handle_pending_setup_changes() + if not ret: + self.found_errors[self.get_env()] = True def load_metadata(self): parser = ScanMetadataParser(self.inv) diff --git a/app/install/db/constants.json b/app/install/db/constants.json index e456873..7a5b795 100644 --- a/app/install/db/constants.json +++ b/app/install/db/constants.json @@ -402,10 +402,14 @@ { "value" : "running", "label" : "Running" - }, + }, { - "value" : "completed", + "value" : "completed", "label" : "Completed" + }, + { + "value" : "completed_with_errors", + "label" : "Completed with errors" }, { "value" : "failed", diff --git a/app/monitoring/setup/monitoring_handler.py b/app/monitoring/setup/monitoring_handler.py index a1ff864..e4af85d 100644 --- a/app/monitoring/setup/monitoring_handler.py +++ b/app/monitoring/setup/monitoring_handler.py @@ -29,7 +29,7 @@ from utils.inventory_mgr import InventoryMgr from utils.logging.full_logger import FullLogger from utils.mongo_access import MongoAccess from utils.ssh_conn import SshConn -from utils.ssh_connection import SshConnection +from utils.ssh_connection import SshConnection, SshError class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): @@ -55,6 +55,7 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): self.mechanism_drivers = \ self.configuration.environment['mechanism_drivers'] self.env = env + self.had_errors = False self.monitoring_config = self.db.monitoring_config_templates try: self.env_monitoring_config = self.configuration.get('Monitoring') @@ -246,7 +247,7 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): if self.provision < self.provision_levels['files']: if self.provision == self.provision_levels['db']: self.log.info('Monitoring config applied only in DB') - return + return True self.log.info('applying monitoring setup') hosts = {} scripts_to_hosts = {} @@ -254,14 +255,16 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): self.handle_pending_host_setup_changes(host_changes, hosts, scripts_to_hosts) if self.provision < self.provision_levels['deploy']: - return + return True if self.fetch_ssl_files: self.deploy_ssl_files(list(scripts_to_hosts.keys())) for host in scripts_to_hosts.values(): self.deploy_scripts_to_host(host) for host in hosts.values(): self.deploy_config_to_target(host) - self.log.info('done applying monitoring setup') + had_errors = ', with some error(s)' if self.had_errors else '' + self.log.info('done applying monitoring setup{}'.format(had_errors)) + return not self.had_errors def handle_pending_host_setup_changes(self, host_changes, hosts, scripts_to_hosts): @@ -291,9 +294,12 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): remote_path = self.PRODUCTION_CONFIG_DIR if os.path.isfile(local_dir): remote_path += os.path.sep + os.path.basename(local_dir) - self.write_to_server(local_dir, - remote_path=remote_path, - is_container=is_container) + try: + self.write_to_server(local_dir, + remote_path=remote_path, + is_container=is_container) + except SshError: + self.had_errors = True elif is_local_host: # write to production configuration directory on local host self.make_directory(self.PRODUCTION_CONFIG_DIR) @@ -302,7 +308,10 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): # write to remote host prepare dir - use sftp if self.provision < self.provision_levels['deploy']: continue - self.write_to_remote_host(host, changes['local_path']) + try: + self.write_to_remote_host(host, changes['local_path']) + except SshError: + self.had_errors = True def prepare_scripts(self, host, is_server): if self.scripts_prepared_for_host.get(host, False): @@ -332,30 +341,36 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): self.scripts_prepared_for_host[host] = True def deploy_ssl_files(self, hosts: list): - monitoring_server = self.env_monitoring_config['server_ip'] - gateway_host = SshConn.get_gateway_host(hosts[0]) - temp_dir = tempfile.TemporaryDirectory() - for file_path in self.fetch_ssl_files: - # copy SSL files from the monitoring server - file_name = os.path.basename(file_path) - local_path = os.path.join(temp_dir.name, file_name) - self.get_file(monitoring_server, file_path, local_path) - # first copy the files to the gateway - self.write_to_remote_host(gateway_host, local_path, - remote_path=file_path) - ssl_path = os.path.commonprefix(self.fetch_ssl_files) - for host in hosts: - self.copy_from_gateway_to_host(host, ssl_path, ssl_path) + try: + monitoring_server = self.env_monitoring_config['server_ip'] + gateway_host = SshConn.get_gateway_host(hosts[0]) + temp_dir = tempfile.TemporaryDirectory() + for file_path in self.fetch_ssl_files: + # copy SSL files from the monitoring server + file_name = os.path.basename(file_path) + local_path = os.path.join(temp_dir.name, file_name) + self.get_file(monitoring_server, file_path, local_path) + # first copy the files to the gateway + self.write_to_remote_host(gateway_host, local_path, + remote_path=file_path) + ssl_path = os.path.commonprefix(self.fetch_ssl_files) + for host in hosts: + self.copy_from_gateway_to_host(host, ssl_path, ssl_path) + except SshError: + self.had_errors = True def deploy_scripts_to_host(self, host_details): - host = host_details['host'] - is_server = host_details['is_server'] - self.prepare_scripts(host, is_server) - remote_path = self.REMOTE_SCRIPTS_FOLDER - local_path = remote_path + os.path.sep + '*.py' - if is_server: - return # this was done earlier - self.copy_from_gateway_to_host(host, local_path, remote_path) + try: + host = host_details['host'] + is_server = host_details['is_server'] + self.prepare_scripts(host, is_server) + remote_path = self.REMOTE_SCRIPTS_FOLDER + local_path = remote_path + os.path.sep + '*.py' + if is_server: + return # this was done earlier + self.copy_from_gateway_to_host(host, local_path, remote_path) + except SshError: + self.had_errors = True def restart_service(self, host: str = None, service: str = 'sensu-client', @@ -365,24 +380,30 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter): cmd = 'sudo /etc/init.d/{} restart'.format(service) log_msg = msg if msg else 'deploying config to host {}'.format(host) self.log.info(log_msg) - if is_server: - ssh.exec(cmd) - else: - self.run(cmd, ssh_to_host=host, ssh=ssh) + try: + if is_server: + ssh.exec(cmd) + else: + self.run(cmd, ssh_to_host=host, ssh=ssh) + except SshError: + self.had_errors = True def deploy_config_to_target(self, host_details): - host = host_details['host'] - is_local_host = host_details['is_local_host'] - is_container = host_details['is_container'] - is_server = host_details['is_server'] - local_dir = host_details['local_dir'] - if is_container or is_server or not is_local_host: - local_dir = os.path.dirname(local_dir) - if not is_server: - self.move_setup_files_to_remote_host(host, local_dir) - # restart the Sensu client on the remote host, - # so it takes the new setup - self.restart_service(host) + try: + host = host_details['host'] + is_local_host = host_details['is_local_host'] + is_container = host_details['is_container'] + is_server = host_details['is_server'] + local_dir = host_details['local_dir'] + if is_container or is_server or not is_local_host: + local_dir = os.path.dirname(local_dir) + if not is_server: + self.move_setup_files_to_remote_host(host, local_dir) + # restart the Sensu client on the remote host, + # so it takes the new setup + self.restart_service(host) + except SshError: + self.had_errors = True def run_cmd_locally(self, cmd): try: diff --git a/app/test/api/responders_test/test_data/base.py b/app/test/api/responders_test/test_data/base.py index 3dbd6f4..d320340 100644 --- a/app/test/api/responders_test/test_data/base.py +++ b/app/test/api/responders_test/test_data/base.py @@ -89,6 +89,7 @@ CONSTANTS_BY_NAMES = { "pending", "running", "completed", + "completed_with_errors", "failed", "aborted" ], diff --git a/app/utils/constants.py b/app/utils/constants.py index 7aa0343..44850b3 100644 --- a/app/utils/constants.py +++ b/app/utils/constants.py @@ -22,6 +22,7 @@ class ScanStatus(StringEnum): PENDING = "pending" RUNNING = "running" COMPLETED = "completed" + COMPLETED_WITH_ERRORS = "completed_with_errors" FAILED = "failed" diff --git a/app/utils/ssh_connection.py b/app/utils/ssh_connection.py index b0f202a..e9dd39a 100644 --- a/app/utils/ssh_connection.py +++ b/app/utils/ssh_connection.py @@ -12,7 +12,10 @@ import os import paramiko from utils.binary_converter import BinaryConverter +from discover.scan_error import ScanError +class SshError(Exception): + pass class SshConnection(BinaryConverter): connections = {} @@ -117,6 +120,7 @@ class SshConnection(BinaryConverter): else self.DEFAULT_PORT, password=self.pwd, timeout=30) else: + port = None try: port = self.port if self.port is not None else self.DEFAULT_PORT self.ssh_client.connect(self.host, @@ -146,12 +150,12 @@ class SshConnection(BinaryConverter): err_lines = [l for l in err.splitlines() if 'Loaded plugin: ' not in l] if err_lines: - self.log.error("CLI access: \n" + - "Host: {}\nCommand: {}\nError: {}\n". - format(self.host, cmd, err)) + msg = "CLI access: \nHost: {}\nCommand: {}\nError: {}\n" + msg = msg.format(self.host, cmd, err) + self.log.error(msg) stderr.close() stdout.close() - return "" + raise SshError(msg) ret = self.binary2str(stdout.read()) stderr.close() stdout.close() @@ -165,11 +169,11 @@ class SshConnection(BinaryConverter): try: self.ftp.put(local_path, remote_path) except IOError as e: - self.log.error('SFTP copy_file failed to copy file: ' + - 'local: ' + local_path + - ', remote host: ' + self.host + - ', error: ' + str(e)) - return str(e) + msg = 'SFTP copy_file failed to copy file: ' \ + 'local: {}, remote host: {}, error: {}' \ + .format(local_path, self.host, str(e)) + self.log.error(msg) + raise SshError(msg) try: remote_file = self.ftp.file(remote_path, 'a+') except IOError as e: @@ -201,11 +205,11 @@ class SshConnection(BinaryConverter): try: self.ftp.get(remote_path, local_path) except IOError as e: - self.log.error('SFTP copy_file_from_remote failed to copy file: ' - 'remote host: {}, ' - 'remote_path: {}, local: {}, error: {}' - .format(self.host, remote_path, local_path, str(e))) - return str(e) + msg = 'SFTP copy_file_from_remote failed to copy file: ' \ + 'remote host: {}, remote_path: {}, local: {}, error: {}' + msg = msg.format(self.host, remote_path, local_path, str(e)) + self.log.error(msg) + raise SshError(msg) self.log.info('SFTP copy_file_from_remote success: host={},{} -> {}'. format(self.host, remote_path, local_path)) return '' diff --git a/ui/imports/api/constants/data/scans-statuses.js b/ui/imports/api/constants/data/scans-statuses.js index 778f256..d61c8f3 100644 --- a/ui/imports/api/constants/data/scans-statuses.js +++ b/ui/imports/api/constants/data/scans-statuses.js @@ -18,6 +18,9 @@ export const Statuses = [{ }, { value: 'completed', label: 'Completed', +}, { + value: 'completed_with_errors', + label: 'Completed with errors', }, { value: 'failed', label: 'Failed', -- cgit 1.2.3-korg