aboutsummaryrefslogtreecommitdiffstats
path: root/app
diff options
context:
space:
mode:
authoryayogev <yaronyogev@gmail.com>2017-08-31 16:45:23 +0300
committeryayogev <yaronyogev@gmail.com>2017-08-31 16:45:23 +0300
commit692489cc50c8025ede1646627a7a583a4feb3798 (patch)
tree531a2f2928ba8c9067265fa0c872cc6f016876b8 /app
parentbc3767bad9f9f9cfb3f3f2c8871a81603e650df0 (diff)
US2876 handle SSH errors
ido not stop, but report as 'completed with errors' if there were any errors in SSH calls Change-Id: Ice7e6c4324686adc2d9eec27a9b6187f0fe6808f Signed-off-by: yayogev <yaronyogev@gmail.com>
Diffstat (limited to 'app')
-rwxr-xr-xapp/discover/scan.py6
-rw-r--r--app/discover/scan_manager.py20
-rw-r--r--app/discover/scanner.py16
-rw-r--r--app/install/db/constants.json8
-rw-r--r--app/monitoring/setup/monitoring_handler.py113
-rw-r--r--app/test/api/responders_test/test_data/base.py1
-rw-r--r--app/utils/constants.py1
-rw-r--r--app/utils/ssh_connection.py32
8 files changed, 124 insertions, 73 deletions
diff --git a/app/discover/scan.py b/app/discover/scan.py
index 72184ec..86ee990 100755
--- a/app/discover/scan.py
+++ b/app/discover/scan.py
@@ -274,6 +274,7 @@ class ScanController(Fetcher):
# generate ScanObject Class and instance.
scanner = Scanner()
scanner.set_env(env_name)
+ scanner.found_errors[env_name] = False
# decide what scanning operations to do
inventory_only = scan_plan.inventory_only
@@ -313,7 +314,10 @@ class ScanController(Fetcher):
except ScanError as e:
return False, "scan error: " + str(e)
SshConnection.disconnect_all()
- return True, 'ok'
+ status = 'ok' if not scanner.found_errors.get(env_name, False) \
+ else 'errors detected'
+ self.log.info('Scan completed, status: {}'.format(status))
+ return True, status
if __name__ == '__main__':
diff --git a/app/discover/scan_manager.py b/app/discover/scan_manager.py
index b6ad782..12dbec0 100644
--- a/app/discover/scan_manager.py
+++ b/app/discover/scan_manager.py
@@ -41,6 +41,8 @@ class ScanManager(Manager):
mongo_config_file=self.args.mongo_config)
self.db_client = None
self.environments_collection = None
+ self.scans_collection = None
+ self.scheduled_scans_collection = None
@staticmethod
def get_args():
@@ -138,8 +140,10 @@ class ScanManager(Manager):
def _fail_scan(self, scan_request: dict):
self._finalize_scan(scan_request, ScanStatus.FAILED, False)
- def _complete_scan(self, scan_request: dict):
- self._finalize_scan(scan_request, ScanStatus.COMPLETED, True)
+ def _complete_scan(self, scan_request: dict, result_message: str):
+ status = ScanStatus.COMPLETED if result_message == 'ok' \
+ else ScanStatus.COMPLETED_WITH_ERRORS
+ self._finalize_scan(scan_request, status, True)
# PyCharm type checker can't reliably check types of document
# noinspection PyTypeChecker
@@ -184,6 +188,7 @@ class ScanManager(Manager):
'scan_only_links': scheduled_scan['scan_only_links'],
'scan_only_cliques': scheduled_scan['scan_only_cliques'],
'submit_timestamp': ts,
+ 'interval': interval,
'environment': scheduled_scan['environment'],
'inventory': 'inventory'
}
@@ -240,8 +245,9 @@ class ScanManager(Manager):
time.sleep(self.interval)
else:
scan_request = results[0]
- if not self.inv.is_feature_supported(scan_request.get('environment'),
- EnvironmentFeatures.SCANNING):
+ env = scan_request.get('environment')
+ scan_feature = EnvironmentFeatures.SCANNING
+ if not self.inv.is_feature_supported(env, scan_feature):
self.log.error("Scanning is not supported for env '{}'"
.format(scan_request.get('environment')))
self._fail_scan(scan_request)
@@ -281,11 +287,11 @@ class ScanManager(Manager):
continue
# update the status and timestamps.
- self.log.info("Request '{}' has been scanned."
- .format(scan_request['_id']))
+ self.log.info("Request '{}' has been scanned. ({})"
+ .format(scan_request['_id'], message))
end_time = datetime.datetime.utcnow()
scan_request['end_timestamp'] = end_time
- self._complete_scan(scan_request)
+ self._complete_scan(scan_request, message)
finally:
self._clean_up()
diff --git a/app/discover/scanner.py b/app/discover/scanner.py
index 1b7cd51..c310ae7 100644
--- a/app/discover/scanner.py
+++ b/app/discover/scanner.py
@@ -25,9 +25,8 @@ from discover.find_links_for_vedges import FindLinksForVedges
from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
from discover.scan_error import ScanError
from discover.scan_metadata_parser import ScanMetadataParser
-from utils.constants import EnvironmentFeatures
from utils.inventory_mgr import InventoryMgr
-from utils.util import ClassResolver
+from utils.ssh_connection import SshError
class Scanner(Fetcher):
@@ -38,6 +37,9 @@ class Scanner(Fetcher):
scan_queue = queue.Queue()
scan_queue_track = {}
+ # keep errors indication per environment
+ found_errors = {}
+
def __init__(self):
"""
Scanner is the base class for scanners.
@@ -71,6 +73,9 @@ class Scanner(Fetcher):
"children": children})
except ValueError:
return False
+ except SshError:
+ # mark the error
+ self.found_errors[self.get_env()] = True
if limit_to_child_id and len(types_children) > 0:
t = types_children[0]
children = t["children"]
@@ -135,6 +140,9 @@ class Scanner(Fetcher):
# It depends on the Fetcher's config.
try:
db_results = fetcher.get(escaped_id)
+ except SshError:
+ self.found_errors[self.get_env()] = True
+ return []
except Exception as e:
self.log.error("Error while scanning : " +
"fetcher=%s, " +
@@ -233,7 +241,9 @@ class Scanner(Fetcher):
clique_scanner.find_cliques()
def deploy_monitoring_setup(self):
- self.inv.monitoring_setup_manager.handle_pending_setup_changes()
+ ret = self.inv.monitoring_setup_manager.handle_pending_setup_changes()
+ if not ret:
+ self.found_errors[self.get_env()] = True
def load_metadata(self):
parser = ScanMetadataParser(self.inv)
diff --git a/app/install/db/constants.json b/app/install/db/constants.json
index e456873..7a5b795 100644
--- a/app/install/db/constants.json
+++ b/app/install/db/constants.json
@@ -402,10 +402,14 @@
{
"value" : "running",
"label" : "Running"
- },
+ },
{
- "value" : "completed",
+ "value" : "completed",
"label" : "Completed"
+ },
+ {
+ "value" : "completed_with_errors",
+ "label" : "Completed with errors"
},
{
"value" : "failed",
diff --git a/app/monitoring/setup/monitoring_handler.py b/app/monitoring/setup/monitoring_handler.py
index a1ff864..e4af85d 100644
--- a/app/monitoring/setup/monitoring_handler.py
+++ b/app/monitoring/setup/monitoring_handler.py
@@ -29,7 +29,7 @@ from utils.inventory_mgr import InventoryMgr
from utils.logging.full_logger import FullLogger
from utils.mongo_access import MongoAccess
from utils.ssh_conn import SshConn
-from utils.ssh_connection import SshConnection
+from utils.ssh_connection import SshConnection, SshError
class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
@@ -55,6 +55,7 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
self.mechanism_drivers = \
self.configuration.environment['mechanism_drivers']
self.env = env
+ self.had_errors = False
self.monitoring_config = self.db.monitoring_config_templates
try:
self.env_monitoring_config = self.configuration.get('Monitoring')
@@ -246,7 +247,7 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
if self.provision < self.provision_levels['files']:
if self.provision == self.provision_levels['db']:
self.log.info('Monitoring config applied only in DB')
- return
+ return True
self.log.info('applying monitoring setup')
hosts = {}
scripts_to_hosts = {}
@@ -254,14 +255,16 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
self.handle_pending_host_setup_changes(host_changes, hosts,
scripts_to_hosts)
if self.provision < self.provision_levels['deploy']:
- return
+ return True
if self.fetch_ssl_files:
self.deploy_ssl_files(list(scripts_to_hosts.keys()))
for host in scripts_to_hosts.values():
self.deploy_scripts_to_host(host)
for host in hosts.values():
self.deploy_config_to_target(host)
- self.log.info('done applying monitoring setup')
+ had_errors = ', with some error(s)' if self.had_errors else ''
+ self.log.info('done applying monitoring setup{}'.format(had_errors))
+ return not self.had_errors
def handle_pending_host_setup_changes(self, host_changes, hosts,
scripts_to_hosts):
@@ -291,9 +294,12 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
remote_path = self.PRODUCTION_CONFIG_DIR
if os.path.isfile(local_dir):
remote_path += os.path.sep + os.path.basename(local_dir)
- self.write_to_server(local_dir,
- remote_path=remote_path,
- is_container=is_container)
+ try:
+ self.write_to_server(local_dir,
+ remote_path=remote_path,
+ is_container=is_container)
+ except SshError:
+ self.had_errors = True
elif is_local_host:
# write to production configuration directory on local host
self.make_directory(self.PRODUCTION_CONFIG_DIR)
@@ -302,7 +308,10 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
# write to remote host prepare dir - use sftp
if self.provision < self.provision_levels['deploy']:
continue
- self.write_to_remote_host(host, changes['local_path'])
+ try:
+ self.write_to_remote_host(host, changes['local_path'])
+ except SshError:
+ self.had_errors = True
def prepare_scripts(self, host, is_server):
if self.scripts_prepared_for_host.get(host, False):
@@ -332,30 +341,36 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
self.scripts_prepared_for_host[host] = True
def deploy_ssl_files(self, hosts: list):
- monitoring_server = self.env_monitoring_config['server_ip']
- gateway_host = SshConn.get_gateway_host(hosts[0])
- temp_dir = tempfile.TemporaryDirectory()
- for file_path in self.fetch_ssl_files:
- # copy SSL files from the monitoring server
- file_name = os.path.basename(file_path)
- local_path = os.path.join(temp_dir.name, file_name)
- self.get_file(monitoring_server, file_path, local_path)
- # first copy the files to the gateway
- self.write_to_remote_host(gateway_host, local_path,
- remote_path=file_path)
- ssl_path = os.path.commonprefix(self.fetch_ssl_files)
- for host in hosts:
- self.copy_from_gateway_to_host(host, ssl_path, ssl_path)
+ try:
+ monitoring_server = self.env_monitoring_config['server_ip']
+ gateway_host = SshConn.get_gateway_host(hosts[0])
+ temp_dir = tempfile.TemporaryDirectory()
+ for file_path in self.fetch_ssl_files:
+ # copy SSL files from the monitoring server
+ file_name = os.path.basename(file_path)
+ local_path = os.path.join(temp_dir.name, file_name)
+ self.get_file(monitoring_server, file_path, local_path)
+ # first copy the files to the gateway
+ self.write_to_remote_host(gateway_host, local_path,
+ remote_path=file_path)
+ ssl_path = os.path.commonprefix(self.fetch_ssl_files)
+ for host in hosts:
+ self.copy_from_gateway_to_host(host, ssl_path, ssl_path)
+ except SshError:
+ self.had_errors = True
def deploy_scripts_to_host(self, host_details):
- host = host_details['host']
- is_server = host_details['is_server']
- self.prepare_scripts(host, is_server)
- remote_path = self.REMOTE_SCRIPTS_FOLDER
- local_path = remote_path + os.path.sep + '*.py'
- if is_server:
- return # this was done earlier
- self.copy_from_gateway_to_host(host, local_path, remote_path)
+ try:
+ host = host_details['host']
+ is_server = host_details['is_server']
+ self.prepare_scripts(host, is_server)
+ remote_path = self.REMOTE_SCRIPTS_FOLDER
+ local_path = remote_path + os.path.sep + '*.py'
+ if is_server:
+ return # this was done earlier
+ self.copy_from_gateway_to_host(host, local_path, remote_path)
+ except SshError:
+ self.had_errors = True
def restart_service(self, host: str = None,
service: str = 'sensu-client',
@@ -365,24 +380,30 @@ class MonitoringHandler(MongoAccess, CliAccess, BinaryConverter):
cmd = 'sudo /etc/init.d/{} restart'.format(service)
log_msg = msg if msg else 'deploying config to host {}'.format(host)
self.log.info(log_msg)
- if is_server:
- ssh.exec(cmd)
- else:
- self.run(cmd, ssh_to_host=host, ssh=ssh)
+ try:
+ if is_server:
+ ssh.exec(cmd)
+ else:
+ self.run(cmd, ssh_to_host=host, ssh=ssh)
+ except SshError:
+ self.had_errors = True
def deploy_config_to_target(self, host_details):
- host = host_details['host']
- is_local_host = host_details['is_local_host']
- is_container = host_details['is_container']
- is_server = host_details['is_server']
- local_dir = host_details['local_dir']
- if is_container or is_server or not is_local_host:
- local_dir = os.path.dirname(local_dir)
- if not is_server:
- self.move_setup_files_to_remote_host(host, local_dir)
- # restart the Sensu client on the remote host,
- # so it takes the new setup
- self.restart_service(host)
+ try:
+ host = host_details['host']
+ is_local_host = host_details['is_local_host']
+ is_container = host_details['is_container']
+ is_server = host_details['is_server']
+ local_dir = host_details['local_dir']
+ if is_container or is_server or not is_local_host:
+ local_dir = os.path.dirname(local_dir)
+ if not is_server:
+ self.move_setup_files_to_remote_host(host, local_dir)
+ # restart the Sensu client on the remote host,
+ # so it takes the new setup
+ self.restart_service(host)
+ except SshError:
+ self.had_errors = True
def run_cmd_locally(self, cmd):
try:
diff --git a/app/test/api/responders_test/test_data/base.py b/app/test/api/responders_test/test_data/base.py
index 3dbd6f4..d320340 100644
--- a/app/test/api/responders_test/test_data/base.py
+++ b/app/test/api/responders_test/test_data/base.py
@@ -89,6 +89,7 @@ CONSTANTS_BY_NAMES = {
"pending",
"running",
"completed",
+ "completed_with_errors",
"failed",
"aborted"
],
diff --git a/app/utils/constants.py b/app/utils/constants.py
index 7aa0343..44850b3 100644
--- a/app/utils/constants.py
+++ b/app/utils/constants.py
@@ -22,6 +22,7 @@ class ScanStatus(StringEnum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
+ COMPLETED_WITH_ERRORS = "completed_with_errors"
FAILED = "failed"
diff --git a/app/utils/ssh_connection.py b/app/utils/ssh_connection.py
index b0f202a..e9dd39a 100644
--- a/app/utils/ssh_connection.py
+++ b/app/utils/ssh_connection.py
@@ -12,7 +12,10 @@ import os
import paramiko
from utils.binary_converter import BinaryConverter
+from discover.scan_error import ScanError
+class SshError(Exception):
+ pass
class SshConnection(BinaryConverter):
connections = {}
@@ -117,6 +120,7 @@ class SshConnection(BinaryConverter):
else self.DEFAULT_PORT,
password=self.pwd, timeout=30)
else:
+ port = None
try:
port = self.port if self.port is not None else self.DEFAULT_PORT
self.ssh_client.connect(self.host,
@@ -146,12 +150,12 @@ class SshConnection(BinaryConverter):
err_lines = [l for l in err.splitlines()
if 'Loaded plugin: ' not in l]
if err_lines:
- self.log.error("CLI access: \n" +
- "Host: {}\nCommand: {}\nError: {}\n".
- format(self.host, cmd, err))
+ msg = "CLI access: \nHost: {}\nCommand: {}\nError: {}\n"
+ msg = msg.format(self.host, cmd, err)
+ self.log.error(msg)
stderr.close()
stdout.close()
- return ""
+ raise SshError(msg)
ret = self.binary2str(stdout.read())
stderr.close()
stdout.close()
@@ -165,11 +169,11 @@ class SshConnection(BinaryConverter):
try:
self.ftp.put(local_path, remote_path)
except IOError as e:
- self.log.error('SFTP copy_file failed to copy file: ' +
- 'local: ' + local_path +
- ', remote host: ' + self.host +
- ', error: ' + str(e))
- return str(e)
+ msg = 'SFTP copy_file failed to copy file: ' \
+ 'local: {}, remote host: {}, error: {}' \
+ .format(local_path, self.host, str(e))
+ self.log.error(msg)
+ raise SshError(msg)
try:
remote_file = self.ftp.file(remote_path, 'a+')
except IOError as e:
@@ -201,11 +205,11 @@ class SshConnection(BinaryConverter):
try:
self.ftp.get(remote_path, local_path)
except IOError as e:
- self.log.error('SFTP copy_file_from_remote failed to copy file: '
- 'remote host: {}, '
- 'remote_path: {}, local: {}, error: {}'
- .format(self.host, remote_path, local_path, str(e)))
- return str(e)
+ msg = 'SFTP copy_file_from_remote failed to copy file: ' \
+ 'remote host: {}, remote_path: {}, local: {}, error: {}'
+ msg = msg.format(self.host, remote_path, local_path, str(e))
+ self.log.error(msg)
+ raise SshError(msg)
self.log.info('SFTP copy_file_from_remote success: host={},{} -> {}'.
format(self.host, remote_path, local_path))
return ''