aboutsummaryrefslogtreecommitdiffstats
path: root/app
diff options
context:
space:
mode:
Diffstat (limited to 'app')
-rw-r--r--app/config/link_finders.json12
-rw-r--r--app/discover/events/event_interface_add.py4
-rw-r--r--app/discover/events/event_port_add.py8
-rw-r--r--app/discover/events/event_router_add.py4
-rw-r--r--app/discover/events/event_router_update.py2
-rw-r--r--app/discover/events/event_subnet_add.py5
-rw-r--r--app/discover/events/event_subnet_update.py2
-rw-r--r--app/discover/link_finders/__init__.py0
-rw-r--r--app/discover/link_finders/find_links.py (renamed from app/discover/find_links.py)0
-rw-r--r--app/discover/link_finders/find_links_for_instance_vnics.py (renamed from app/discover/find_links_for_instance_vnics.py)2
-rw-r--r--app/discover/link_finders/find_links_for_oteps.py (renamed from app/discover/find_links_for_oteps.py)2
-rw-r--r--app/discover/link_finders/find_links_for_pnics.py (renamed from app/discover/find_links_for_pnics.py)2
-rw-r--r--app/discover/link_finders/find_links_for_vconnectors.py (renamed from app/discover/find_links_for_vconnectors.py)2
-rw-r--r--app/discover/link_finders/find_links_for_vedges.py (renamed from app/discover/find_links_for_vedges.py)2
-rw-r--r--app/discover/link_finders/find_links_for_vservice_vnics.py (renamed from app/discover/find_links_for_vservice_vnics.py)2
-rw-r--r--app/discover/link_finders/find_links_metadata_parser.py79
-rw-r--r--app/discover/scanner.py35
-rw-r--r--app/test/event_based_scan/test_port_add.py7
-rw-r--r--app/test/event_based_scan/test_subnet_add.py4
-rw-r--r--app/test/scan/test_scan_controller.py9
20 files changed, 136 insertions, 47 deletions
diff --git a/app/config/link_finders.json b/app/config/link_finders.json
new file mode 100644
index 0000000..2368333
--- /dev/null
+++ b/app/config/link_finders.json
@@ -0,0 +1,12 @@
+{
+ "finders_package": "discover.link_finders",
+ "base_finder": "FindLinks",
+ "link_finders": [
+ "FindLinksForInstanceVnics",
+ "FindLinksForOteps",
+ "FindLinksForPnics",
+ "FindLinksForVconnectors",
+ "FindLinksForVedges",
+ "FindLinksForVserviceVnics"
+ ]
+} \ No newline at end of file
diff --git a/app/discover/events/event_interface_add.py b/app/discover/events/event_interface_add.py
index 2c7af13..698559c 100644
--- a/app/discover/events/event_interface_add.py
+++ b/app/discover/events/event_interface_add.py
@@ -8,7 +8,6 @@
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
import time
-
from functools import partial
from discover.events.event_base import EventBase, EventResult
@@ -17,7 +16,8 @@ from discover.events.event_subnet_add import EventSubnetAdd
from discover.fetchers.api.api_access import ApiAccess
from discover.fetchers.api.api_fetch_regions import ApiFetchRegions
from discover.fetchers.cli.cli_fetch_host_vservice import CliFetchHostVservice
-from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
+from discover.link_finders.find_links_for_vservice_vnics import \
+ FindLinksForVserviceVnics
from discover.scanner import Scanner
from utils.util import decode_router_id, encode_router_id
diff --git a/app/discover/events/event_port_add.py b/app/discover/events/event_port_add.py
index 63a5e80..9220015 100644
--- a/app/discover/events/event_port_add.py
+++ b/app/discover/events/event_port_add.py
@@ -12,10 +12,12 @@ import datetime
from discover.events.event_base import EventBase, EventResult
from discover.fetchers.api.api_fetch_host_instances import ApiFetchHostInstances
from discover.fetchers.cli.cli_fetch_instance_vnics import CliFetchInstanceVnics
-from discover.fetchers.cli.cli_fetch_instance_vnics_vpp import CliFetchInstanceVnicsVpp
+from discover.fetchers.cli.cli_fetch_instance_vnics_vpp import \
+ CliFetchInstanceVnicsVpp
from discover.fetchers.cli.cli_fetch_vservice_vnics import CliFetchVserviceVnics
-from discover.find_links_for_instance_vnics import FindLinksForInstanceVnics
-from discover.find_links_for_vedges import FindLinksForVedges
+from discover.link_finders.find_links_for_instance_vnics import \
+ FindLinksForInstanceVnics
+from discover.link_finders.find_links_for_vedges import FindLinksForVedges
from discover.scanner import Scanner
diff --git a/app/discover/events/event_router_add.py b/app/discover/events/event_router_add.py
index 20e07e5..3c1c9e2 100644
--- a/app/discover/events/event_router_add.py
+++ b/app/discover/events/event_router_add.py
@@ -8,14 +8,14 @@
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
import datetime
-
from functools import partial
from discover.events.event_base import EventBase, EventResult
from discover.events.event_port_add import EventPortAdd
from discover.events.event_subnet_add import EventSubnetAdd
from discover.fetchers.cli.cli_fetch_host_vservice import CliFetchHostVservice
-from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
+from discover.link_finders.find_links_for_vservice_vnics import \
+ FindLinksForVserviceVnics
from discover.scanner import Scanner
from utils.util import decode_router_id, encode_router_id
diff --git a/app/discover/events/event_router_update.py b/app/discover/events/event_router_update.py
index 8dd53f0..cfbbf58 100644
--- a/app/discover/events/event_router_update.py
+++ b/app/discover/events/event_router_update.py
@@ -11,7 +11,7 @@ from discover.events.event_base import EventBase, EventResult
from discover.events.event_port_delete import EventPortDelete
from discover.events.event_router_add import EventRouterAdd
from discover.fetchers.cli.cli_fetch_host_vservice import CliFetchHostVservice
-from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
+from discover.link_finders.find_links_for_vservice_vnics import FindLinksForVserviceVnics
from discover.scanner import Scanner
from utils.util import encode_router_id
diff --git a/app/discover/events/event_subnet_add.py b/app/discover/events/event_subnet_add.py
index a33f7cf..fcae5fd 100644
--- a/app/discover/events/event_subnet_add.py
+++ b/app/discover/events/event_subnet_add.py
@@ -15,8 +15,9 @@ from discover.fetchers.api.api_access import ApiAccess
from discover.fetchers.api.api_fetch_port import ApiFetchPort
from discover.fetchers.api.api_fetch_regions import ApiFetchRegions
from discover.fetchers.db.db_fetch_port import DbFetchPort
-from discover.find_links_for_pnics import FindLinksForPnics
-from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
+from discover.link_finders.find_links_for_pnics import FindLinksForPnics
+from discover.link_finders.find_links_for_vservice_vnics import \
+ FindLinksForVserviceVnics
from discover.scanner import Scanner
diff --git a/app/discover/events/event_subnet_update.py b/app/discover/events/event_subnet_update.py
index 26d1984..3529f0d 100644
--- a/app/discover/events/event_subnet_update.py
+++ b/app/discover/events/event_subnet_update.py
@@ -14,7 +14,7 @@ from discover.events.event_subnet_add import EventSubnetAdd
from discover.fetchers.api.api_access import ApiAccess
from discover.fetchers.api.api_fetch_regions import ApiFetchRegions
from discover.fetchers.db.db_fetch_port import DbFetchPort
-from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
+from discover.link_finders.find_links_for_vservice_vnics import FindLinksForVserviceVnics
from discover.scanner import Scanner
diff --git a/app/discover/link_finders/__init__.py b/app/discover/link_finders/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/app/discover/link_finders/__init__.py
diff --git a/app/discover/find_links.py b/app/discover/link_finders/find_links.py
index d234479..d234479 100644
--- a/app/discover/find_links.py
+++ b/app/discover/link_finders/find_links.py
diff --git a/app/discover/find_links_for_instance_vnics.py b/app/discover/link_finders/find_links_for_instance_vnics.py
index 55f7bfc..7e0273d 100644
--- a/app/discover/find_links_for_instance_vnics.py
+++ b/app/discover/link_finders/find_links_for_instance_vnics.py
@@ -7,7 +7,7 @@
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
-from discover.find_links import FindLinks
+from discover.link_finders.find_links import FindLinks
class FindLinksForInstanceVnics(FindLinks):
diff --git a/app/discover/find_links_for_oteps.py b/app/discover/link_finders/find_links_for_oteps.py
index 585dc19..b5d1667 100644
--- a/app/discover/find_links_for_oteps.py
+++ b/app/discover/link_finders/find_links_for_oteps.py
@@ -7,7 +7,7 @@
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
-from discover.find_links import FindLinks
+from discover.link_finders.find_links import FindLinks
class FindLinksForOteps(FindLinks):
diff --git a/app/discover/find_links_for_pnics.py b/app/discover/link_finders/find_links_for_pnics.py
index 98d481c..1f02426 100644
--- a/app/discover/find_links_for_pnics.py
+++ b/app/discover/link_finders/find_links_for_pnics.py
@@ -9,7 +9,7 @@
###############################################################################
import re
-from discover.find_links import FindLinks
+from discover.link_finders.find_links import FindLinks
from utils.util import decode_aci_dn
diff --git a/app/discover/find_links_for_vconnectors.py b/app/discover/link_finders/find_links_for_vconnectors.py
index ab96ebe..edb351a 100644
--- a/app/discover/find_links_for_vconnectors.py
+++ b/app/discover/link_finders/find_links_for_vconnectors.py
@@ -7,7 +7,7 @@
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
-from discover.find_links import FindLinks
+from discover.link_finders.find_links import FindLinks
class FindLinksForVconnectors(FindLinks):
diff --git a/app/discover/find_links_for_vedges.py b/app/discover/link_finders/find_links_for_vedges.py
index acafceb..f9719b4 100644
--- a/app/discover/find_links_for_vedges.py
+++ b/app/discover/link_finders/find_links_for_vedges.py
@@ -7,7 +7,7 @@
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
-from discover.find_links import FindLinks
+from discover.link_finders.find_links import FindLinks
class FindLinksForVedges(FindLinks):
diff --git a/app/discover/find_links_for_vservice_vnics.py b/app/discover/link_finders/find_links_for_vservice_vnics.py
index 3b8f32d..ca9bc4a 100644
--- a/app/discover/find_links_for_vservice_vnics.py
+++ b/app/discover/link_finders/find_links_for_vservice_vnics.py
@@ -7,7 +7,7 @@
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
-from discover.find_links import FindLinks
+from discover.link_finders.find_links import FindLinks
class FindLinksForVserviceVnics(FindLinks):
diff --git a/app/discover/link_finders/find_links_metadata_parser.py b/app/discover/link_finders/find_links_metadata_parser.py
new file mode 100644
index 0000000..a1e290c
--- /dev/null
+++ b/app/discover/link_finders/find_links_metadata_parser.py
@@ -0,0 +1,79 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from utils.metadata_parser import MetadataParser
+from utils.util import ClassResolver
+
+
+class FindLinksMetadataParser(MetadataParser):
+
+ FINDERS_FILE = "link_finders.json"
+
+ FINDERS_PACKAGE = "finders_package"
+ BASE_FINDER = "base_finder"
+ LINK_FINDERS = "link_finders"
+
+ def __init__(self):
+ super().__init__()
+ self.finders_package = None
+ self.base_finder_class = None
+ self.base_finder = None
+ self.link_finders = []
+
+ def validate_link_finder(self, finder_class):
+ try:
+ module_name = ClassResolver.get_module_file_by_class_name(finder_class)
+ instance = ClassResolver\
+ .get_instance_of_class(package_name=self.finders_package,
+ module_name=module_name,
+ class_name=finder_class)
+
+ except ValueError:
+ instance = None
+
+ if instance:
+ if isinstance(instance, self.base_finder.__class__):
+ self.link_finders.append(instance)
+ else:
+ self.add_error('Link finder "{}" should subclass '
+ 'base link finder "{}"'
+ .format(finder_class, self.base_finder_class))
+ else:
+ self.add_error('Failed to import link finder class "{}"'
+ .format(finder_class))
+
+ def validate_metadata(self, metadata: dict):
+ super().validate_metadata(metadata)
+ self.finders_package = metadata[self.FINDERS_PACKAGE]
+ self.base_finder_class = metadata[self.BASE_FINDER]
+ base_finder_module = ClassResolver\
+ .get_module_file_by_class_name(self.base_finder_class)
+
+ try:
+ self.base_finder = ClassResolver\
+ .get_instance_of_class(package_name=self.finders_package,
+ module_name=base_finder_module,
+ class_name=self.base_finder_class)
+ except ValueError:
+ self.base_finder = None
+
+ if not self.base_finder:
+ self.add_error("Couldn't create base link finder instance"
+ "for class name '{}'".format(self.base_finder_class))
+ return False
+
+ for link_finder in metadata[self.LINK_FINDERS]:
+ self.validate_link_finder(finder_class=link_finder)
+ metadata[self.LINK_FINDERS] = self.link_finders
+
+ return len(self.errors) == 0
+
+ def get_required_fields(self) -> list:
+ return [self.FINDERS_PACKAGE, self.BASE_FINDER, self.LINK_FINDERS]
+
diff --git a/app/discover/scanner.py b/app/discover/scanner.py
index c310ae7..3a0b2a5 100644
--- a/app/discover/scanner.py
+++ b/app/discover/scanner.py
@@ -10,19 +10,15 @@
# base class for scanners
import json
-import queue
import os
+import queue
import traceback
from discover.clique_finder import CliqueFinder
from discover.configuration import Configuration
from discover.fetcher import Fetcher
-from discover.find_links_for_instance_vnics import FindLinksForInstanceVnics
-from discover.find_links_for_oteps import FindLinksForOteps
-from discover.find_links_for_pnics import FindLinksForPnics
-from discover.find_links_for_vconnectors import FindLinksForVconnectors
-from discover.find_links_for_vedges import FindLinksForVedges
-from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
+from discover.link_finders.find_links_metadata_parser import \
+ FindLinksMetadataParser
from discover.scan_error import ScanError
from discover.scan_metadata_parser import ScanMetadataParser
from utils.inventory_mgr import InventoryMgr
@@ -49,7 +45,9 @@ class Scanner(Fetcher):
self.inv = InventoryMgr()
self.scanners_package = None
self.scanners = {}
- self.load_metadata()
+ self.link_finders = []
+ self.load_scanners_metadata()
+ self.load_link_finders_metadata()
def scan(self, scanner_type, obj, id_field="id",
limit_to_child_id=None, limit_to_child_type=None):
@@ -223,15 +221,7 @@ class Scanner(Fetcher):
def scan_links(self):
self.log.info("scanning for links")
- fetchers_implementing_add_links = [
- FindLinksForPnics(),
- FindLinksForInstanceVnics(),
- FindLinksForVserviceVnics(),
- FindLinksForVconnectors(),
- FindLinksForVedges(),
- FindLinksForOteps()
- ]
- for fetcher in fetchers_implementing_add_links:
+ for fetcher in self.link_finders:
fetcher.set_env(self.get_env())
fetcher.add_links()
@@ -245,7 +235,7 @@ class Scanner(Fetcher):
if not ret:
self.found_errors[self.get_env()] = True
- def load_metadata(self):
+ def load_scanners_metadata(self):
parser = ScanMetadataParser(self.inv)
conf = self.config.get_env_config()
scanners_file = os.path.join(conf.get('app_path', '/etc/calipso'),
@@ -256,6 +246,15 @@ class Scanner(Fetcher):
self.scanners_package = metadata[ScanMetadataParser.SCANNERS_PACKAGE]
self.scanners = metadata[ScanMetadataParser.SCANNERS]
+ def load_link_finders_metadata(self):
+ parser = FindLinksMetadataParser()
+ conf = self.config.get_env_config()
+ finders_file = os.path.join(conf.get('app_path', '/etc/calipso'),
+ 'config',
+ FindLinksMetadataParser.FINDERS_FILE)
+ metadata = parser.parse_metadata_file(finders_file)
+ self.link_finders = metadata[FindLinksMetadataParser.LINK_FINDERS]
+
def get_scanner_package(self):
return self.scanners_package
diff --git a/app/test/event_based_scan/test_port_add.py b/app/test/event_based_scan/test_port_add.py
index 58926a9..caa8401 100644
--- a/app/test/event_based_scan/test_port_add.py
+++ b/app/test/event_based_scan/test_port_add.py
@@ -7,14 +7,9 @@
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
from discover.events.event_port_add import EventPortAdd
-from discover.fetchers.api.api_fetch_host_instances import ApiFetchHostInstances
-from discover.fetchers.cli.cli_fetch_instance_vnics import CliFetchInstanceVnics
-from discover.find_links_for_instance_vnics import FindLinksForInstanceVnics
-from discover.find_links_for_vedges import FindLinksForVedges
-from discover.scanner import Scanner
from test.event_based_scan.test_data.event_payload_port_add import \
EVENT_PAYLOAD_PORT_INSTANCE_ADD, NETWORK_DOC, \
INSTANCE_DOC, INSTANCES_ROOT, VNIC_DOCS, INSTANCE_DOCS, PORTS_FOLDER, \
diff --git a/app/test/event_based_scan/test_subnet_add.py b/app/test/event_based_scan/test_subnet_add.py
index ce934f3..b9145a5 100644
--- a/app/test/event_based_scan/test_subnet_add.py
+++ b/app/test/event_based_scan/test_subnet_add.py
@@ -7,12 +7,10 @@
# which accompanies this distribution, and is available at #
# http://www.apache.org/licenses/LICENSE-2.0 #
###############################################################################
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
from discover.events.event_subnet_add import EventSubnetAdd
from discover.fetchers.api.api_access import ApiAccess
-from discover.find_links_for_pnics import FindLinksForPnics
-from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics
from test.event_based_scan.test_data.event_payload_subnet_add import \
EVENT_PAYLOAD_SUBNET_ADD, \
EVENT_PAYLOAD_REGION, NETWORK_DOC, HOST_DOC, PORT_DOC
diff --git a/app/test/scan/test_scan_controller.py b/app/test/scan/test_scan_controller.py
index f3bcc9a..f903e11 100644
--- a/app/test/scan/test_scan_controller.py
+++ b/app/test/scan/test_scan_controller.py
@@ -140,20 +140,23 @@ class TestScanController(TestScan):
deploy_monitoring_setup_count)
def prepare_scan_mocks(self):
- self.load_metadata = Scanner.load_metadata
+ self.load_link_finders_metadata = Scanner.load_scanners_metadata
+ self.load_scanners_metadata = Scanner.load_scanners_metadata
self.scan = Scanner.scan
self.scan_links = Scanner.scan_links
self.scan_cliques = Scanner.scan_cliques
self.deploy_monitoring_setup = Scanner.deploy_monitoring_setup
- Scanner.load_metadata = MagicMock()
+ Scanner.load_link_finders_metadata = MagicMock()
+ Scanner.load_scanners_metadata = MagicMock()
Scanner.scan = MagicMock()
Scanner.scan_links = MagicMock()
Scanner.scan_cliques = MagicMock()
Scanner.deploy_monitoring_setup = MagicMock()
def reset_methods(self):
- Scanner.load_metadata = self.load_metadata
+ Scanner.load_link_finders_metadata = self.load_link_finders_metadata
+ Scanner.load_scanners_metadata = self.load_scanners_metadata
Scanner.scan = self.scan
Scanner.scan_links = self.scan_links
Scanner.scan_cliques = self.scan_cliques