US2876 handle SSH errors

ido not stop, but report as 'completed with errors'
if there were any errors in SSH calls

Change-Id: Ice7e6c4324686adc2d9eec27a9b6187f0fe6808f
Signed-off-by: yayogev <yaronyogev@gmail.com>

3 files changed, 31 insertions, 11 deletions

diff --git a/app/discover/scan.py b/app/discover/scan.py
index 72184ec..86ee990 100755
--- a/app/discover/scan.py
+++ b/app/discover/scan.py
@@ -274,6 +274,7 @@ class ScanController(Fetcher):
         # generate ScanObject Class and instance.
         scanner = Scanner()
         scanner.set_env(env_name)
+        scanner.found_errors[env_name] = False
 
         # decide what scanning operations to do
         inventory_only = scan_plan.inventory_only
@@ -313,7 +314,10 @@ class ScanController(Fetcher):
         except ScanError as e:
             return False, "scan error: " + str(e)
         SshConnection.disconnect_all()
-        return True, 'ok'
+        status = 'ok' if not scanner.found_errors.get(env_name, False) \
+            else 'errors detected'
+        self.log.info('Scan completed, status: {}'.format(status))
+        return True, status
 
 
 if __name__ == '__main__':
diff --git a/app/discover/scan_manager.py b/app/discover/scan_manager.py
index b6ad782..12dbec0 100644
--- a/app/discover/scan_manager.py
+++ b/app/discover/scan_manager.py
@@ -41,6 +41,8 @@ class ScanManager(Manager):
                          mongo_config_file=self.args.mongo_config)
         self.db_client = None
         self.environments_collection = None
+        self.scans_collection = None
+        self.scheduled_scans_collection = None
 
     @staticmethod
     def get_args():
@@ -138,8 +140,10 @@ class ScanManager(Manager):
     def _fail_scan(self, scan_request: dict):
         self._finalize_scan(scan_request, ScanStatus.FAILED, False)
 
-    def _complete_scan(self, scan_request: dict):
-        self._finalize_scan(scan_request, ScanStatus.COMPLETED, True)
+    def _complete_scan(self, scan_request: dict, result_message: str):
+        status = ScanStatus.COMPLETED if result_message == 'ok' \
+            else ScanStatus.COMPLETED_WITH_ERRORS
+        self._finalize_scan(scan_request, status, True)
 
     # PyCharm type checker can't reliably check types of document
     # noinspection PyTypeChecker
@@ -184,6 +188,7 @@ class ScanManager(Manager):
             'scan_only_links': scheduled_scan['scan_only_links'],
             'scan_only_cliques': scheduled_scan['scan_only_cliques'],
             'submit_timestamp': ts,
+            'interval': interval,
             'environment': scheduled_scan['environment'],
             'inventory': 'inventory'
         }
@@ -240,8 +245,9 @@ class ScanManager(Manager):
                     time.sleep(self.interval)
                 else:
                     scan_request = results[0]
-                    if not self.inv.is_feature_supported(scan_request.get('environment'),
-                                                         EnvironmentFeatures.SCANNING):
+                    env = scan_request.get('environment')
+                    scan_feature = EnvironmentFeatures.SCANNING
+                    if not self.inv.is_feature_supported(env, scan_feature):
                         self.log.error("Scanning is not supported for env '{}'"
                                        .format(scan_request.get('environment')))
                         self._fail_scan(scan_request)
@@ -281,11 +287,11 @@ class ScanManager(Manager):
                             continue
 
                         # update the status and timestamps.
-                        self.log.info("Request '{}' has been scanned."
-                                      .format(scan_request['_id']))
+                        self.log.info("Request '{}' has been scanned. ({})"
+                                      .format(scan_request['_id'], message))
                         end_time = datetime.datetime.utcnow()
                         scan_request['end_timestamp'] = end_time
-                        self._complete_scan(scan_request)
+                        self._complete_scan(scan_request, message)
         finally:
             self._clean_up()
 
diff --git a/app/discover/scanner.py b/app/discover/scanner.py
index 1b7cd51..c310ae7 100644
--- a/app/discover/scanner.py
+++ b/app/discover/scanner.py
@@ -25,9 +25,8 @@ from discover.find_links_for_vedges import FindLinksForVedges
 from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
 from discover.scan_error import ScanError
 from discover.scan_metadata_parser import ScanMetadataParser
-from utils.constants import EnvironmentFeatures
 from utils.inventory_mgr import InventoryMgr
-from utils.util import ClassResolver
+from utils.ssh_connection import SshError
 
 
 class Scanner(Fetcher):
@@ -38,6 +37,9 @@ class Scanner(Fetcher):
     scan_queue = queue.Queue()
     scan_queue_track = {}
 
+    # keep errors indication per environment
+    found_errors = {}
+
     def __init__(self):
         """
         Scanner is the base class for scanners.
@@ -71,6 +73,9 @@ class Scanner(Fetcher):
                                       "children": children})
         except ValueError:
             return False
+        except SshError:
+            # mark the error
+            self.found_errors[self.get_env()] = True
         if limit_to_child_id and len(types_children) > 0:
             t = types_children[0]
             children = t["children"]
@@ -135,6 +140,9 @@ class Scanner(Fetcher):
         # It depends on the Fetcher's config.
         try:
             db_results = fetcher.get(escaped_id)
+        except SshError:
+            self.found_errors[self.get_env()] = True
+            return []
         except Exception as e:
             self.log.error("Error while scanning : " +
                            "fetcher=%s, " +
@@ -233,7 +241,9 @@ class Scanner(Fetcher):
         clique_scanner.find_cliques()
 
     def deploy_monitoring_setup(self):
-        self.inv.monitoring_setup_manager.handle_pending_setup_changes()
+        ret = self.inv.monitoring_setup_manager.handle_pending_setup_changes()
+        if not ret:
+            self.found_errors[self.get_env()] = True
 
     def load_metadata(self):
         parser = ScanMetadataParser(self.inv)