summaryrefslogtreecommitdiffstats
path: root/app/discover/scanner.py
diff options
context:
space:
mode:
Diffstat (limited to 'app/discover/scanner.py')
-rw-r--r--app/discover/scanner.py253
1 files changed, 253 insertions, 0 deletions
diff --git a/app/discover/scanner.py b/app/discover/scanner.py
new file mode 100644
index 0000000..1b7cd51
--- /dev/null
+++ b/app/discover/scanner.py
@@ -0,0 +1,253 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+# base class for scanners
+
+import json
+import queue
+import os
+import traceback
+
+from discover.clique_finder import CliqueFinder
+from discover.configuration import Configuration
+from discover.fetcher import Fetcher
+from discover.find_links_for_instance_vnics import FindLinksForInstanceVnics
+from discover.find_links_for_oteps import FindLinksForOteps
+from discover.find_links_for_pnics import FindLinksForPnics
+from discover.find_links_for_vconnectors import FindLinksForVconnectors
+from discover.find_links_for_vedges import FindLinksForVedges
+from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
+from discover.scan_error import ScanError
+from discover.scan_metadata_parser import ScanMetadataParser
+from utils.constants import EnvironmentFeatures
+from utils.inventory_mgr import InventoryMgr
+from utils.util import ClassResolver
+
+
+class Scanner(Fetcher):
+ config = None
+ environment = None
+ env = None
+ root_patern = None
+ scan_queue = queue.Queue()
+ scan_queue_track = {}
+
+ def __init__(self):
+ """
+ Scanner is the base class for scanners.
+ """
+ super().__init__()
+ self.config = Configuration()
+ self.inv = InventoryMgr()
+ self.scanners_package = None
+ self.scanners = {}
+ self.load_metadata()
+
+ def scan(self, scanner_type, obj, id_field="id",
+ limit_to_child_id=None, limit_to_child_type=None):
+ types_to_fetch = self.get_scanner(scanner_type)
+ types_children = []
+ if not limit_to_child_type:
+ limit_to_child_type = []
+ elif isinstance(limit_to_child_type, str):
+ limit_to_child_type = [limit_to_child_type]
+ try:
+ for t in types_to_fetch:
+ if limit_to_child_type and t["type"] not in limit_to_child_type:
+ continue
+ children = self.scan_type(t, obj, id_field)
+ if limit_to_child_id:
+ children = [c for c in children
+ if c[id_field] == limit_to_child_id]
+ if not children:
+ continue
+ types_children.append({"type": t["type"],
+ "children": children})
+ except ValueError:
+ return False
+ if limit_to_child_id and len(types_children) > 0:
+ t = types_children[0]
+ children = t["children"]
+ return children[0]
+ return obj
+
+ def check_type_env(self, type_to_fetch):
+ # check if type is to be run in this environment
+ if "environment_condition" not in type_to_fetch:
+ return True
+ env_cond = type_to_fetch.get("environment_condition", {})
+ if not env_cond:
+ return True
+ if not isinstance(env_cond, dict):
+ self.log.warn('illegal environment_condition given '
+ 'for type {}'.format(type_to_fetch['type']))
+ return True
+ conf = self.config.get_env_config()
+ for attr, required_val in env_cond.items():
+ if attr == "mechanism_drivers":
+ if "mechanism_drivers" not in conf:
+ self.log.warn('illegal environment configuration: '
+ 'missing mechanism_drivers')
+ return False
+ if not isinstance(required_val, list):
+ required_val = [required_val]
+ return bool(set(required_val) & set(conf["mechanism_drivers"]))
+ elif attr not in conf or conf[attr] != required_val:
+ return False
+ # no check failed
+ return True
+
+ def scan_type(self, type_to_fetch, parent, id_field):
+ # check if type is to be run in this environment
+ if not self.check_type_env(type_to_fetch):
+ return []
+
+ if not parent:
+ obj_id = None
+ else:
+ obj_id = str(parent[id_field])
+ if not obj_id or not obj_id.rstrip():
+ raise ValueError("Object missing " + id_field + " attribute")
+
+ # get Fetcher instance
+ fetcher = type_to_fetch["fetcher"]
+ fetcher.set_env(self.get_env())
+
+ # get children_scanner instance
+ children_scanner = type_to_fetch.get("children_scanner")
+
+ escaped_id = fetcher.escape(str(obj_id)) if obj_id else obj_id
+ self.log.info(
+ "scanning : type=%s, parent: (type=%s, name=%s, id=%s)",
+ type_to_fetch["type"],
+ parent.get('type', 'environment'),
+ parent.get('name', ''),
+ escaped_id)
+
+ # fetch OpenStack data from environment by CLI, API or MySQL
+ # or physical devices data from ACI API
+ # It depends on the Fetcher's config.
+ try:
+ db_results = fetcher.get(escaped_id)
+ except Exception as e:
+ self.log.error("Error while scanning : " +
+ "fetcher=%s, " +
+ "type=%s, " +
+ "parent: (type=%s, name=%s, id=%s), " +
+ "error: %s",
+ fetcher.__class__.__name__,
+ type_to_fetch["type"],
+ "environment" if "type" not in parent
+ else parent["type"],
+ "" if "name" not in parent else parent["name"],
+ escaped_id,
+ e)
+ traceback.print_exc()
+ raise ScanError(str(e))
+
+ # format results
+ if isinstance(db_results, dict):
+ results = db_results["rows"] if db_results["rows"] else [db_results]
+ elif isinstance(db_results, str):
+ results = json.loads(db_results)
+ else:
+ results = db_results
+
+ # get child_id_field
+ try:
+ child_id_field = type_to_fetch["object_id_to_use_in_child"]
+ except KeyError:
+ child_id_field = "id"
+
+ environment = self.get_env()
+ children = []
+
+ for o in results:
+ saved = self.inv.save_inventory_object(o,
+ parent=parent,
+ environment=environment,
+ type_to_fetch=type_to_fetch)
+
+ if saved:
+ # add objects into children list.
+ children.append(o)
+
+ # put children scanner into queue
+ if children_scanner:
+ self.queue_for_scan(o, child_id_field, children_scanner)
+ return children
+
+ # scanning queued items, rather than going depth-first (DFS)
+ # this is done to allow collecting all required data for objects
+ # before continuing to next level
+ # for example, get host ID from API os-hypervisors call, so later
+ # we can use this ID in the "os-hypervisors/<ID>/servers" call
+ @staticmethod
+ def queue_for_scan(o, child_id_field, children_scanner):
+ if o["id"] in Scanner.scan_queue_track:
+ return
+ Scanner.scan_queue_track[o["type"] + ";" + o["id"]] = 1
+ Scanner.scan_queue.put({"object": o,
+ "child_id_field": child_id_field,
+ "scanner": children_scanner})
+
+ def run_scan(self, scanner_type, obj, id_field, child_id, child_type):
+ results = self.scan(scanner_type, obj, id_field, child_id, child_type)
+
+ # run children scanner from queue.
+ self.scan_from_queue()
+ return results
+
+ def scan_from_queue(self):
+ while not Scanner.scan_queue.empty():
+ item = Scanner.scan_queue.get()
+ scanner_type = item["scanner"]
+
+ # scan the queued item
+ self.scan(scanner_type, item["object"], item["child_id_field"])
+ self.log.info("Scan complete")
+
+ def scan_links(self):
+ self.log.info("scanning for links")
+ fetchers_implementing_add_links = [
+ FindLinksForPnics(),
+ FindLinksForInstanceVnics(),
+ FindLinksForVserviceVnics(),
+ FindLinksForVconnectors(),
+ FindLinksForVedges(),
+ FindLinksForOteps()
+ ]
+ for fetcher in fetchers_implementing_add_links:
+ fetcher.set_env(self.get_env())
+ fetcher.add_links()
+
+ def scan_cliques(self):
+ clique_scanner = CliqueFinder()
+ clique_scanner.set_env(self.get_env())
+ clique_scanner.find_cliques()
+
+ def deploy_monitoring_setup(self):
+ self.inv.monitoring_setup_manager.handle_pending_setup_changes()
+
+ def load_metadata(self):
+ parser = ScanMetadataParser(self.inv)
+ conf = self.config.get_env_config()
+ scanners_file = os.path.join(conf.get('app_path', '/etc/calipso'),
+ 'config',
+ ScanMetadataParser.SCANNERS_FILE)
+
+ metadata = parser.parse_metadata_file(scanners_file)
+ self.scanners_package = metadata[ScanMetadataParser.SCANNERS_PACKAGE]
+ self.scanners = metadata[ScanMetadataParser.SCANNERS]
+
+ def get_scanner_package(self):
+ return self.scanners_package
+
+ def get_scanner(self, scanner_type: str) -> dict:
+ return self.scanners.get(scanner_type)