From 7b4235dced0c66096638bebd5cc98a631538b0e1 Mon Sep 17 00:00:00 2001 From: Ilia Abashin Date: Fri, 1 Sep 2017 15:51:52 +0300 Subject: Refactored link finders Mappings are now defined in a configuration file and fetched dynamically. Change-Id: I250c22967fc66fc0aca173d4c9d65581d879b5d2 Signed-off-by: Ilia Abashin --- app/config/link_finders.json | 12 ++ app/discover/events/event_interface_add.py | 4 +- app/discover/events/event_port_add.py | 8 +- app/discover/events/event_router_add.py | 4 +- app/discover/events/event_router_update.py | 2 +- app/discover/events/event_subnet_add.py | 5 +- app/discover/events/event_subnet_update.py | 2 +- app/discover/find_links.py | 35 ------ app/discover/find_links_for_instance_vnics.py | 60 ---------- app/discover/find_links_for_oteps.py | 88 -------------- app/discover/find_links_for_pnics.py | 132 --------------------- app/discover/find_links_for_vconnectors.py | 91 -------------- app/discover/find_links_for_vedges.py | 128 -------------------- app/discover/find_links_for_vservice_vnics.py | 57 --------- app/discover/link_finders/__init__.py | 0 app/discover/link_finders/find_links.py | 35 ++++++ .../link_finders/find_links_for_instance_vnics.py | 60 ++++++++++ app/discover/link_finders/find_links_for_oteps.py | 88 ++++++++++++++ app/discover/link_finders/find_links_for_pnics.py | 132 +++++++++++++++++++++ .../link_finders/find_links_for_vconnectors.py | 91 ++++++++++++++ app/discover/link_finders/find_links_for_vedges.py | 128 ++++++++++++++++++++ .../link_finders/find_links_for_vservice_vnics.py | 57 +++++++++ .../link_finders/find_links_metadata_parser.py | 57 +++++++++ app/discover/scanner.py | 35 +++--- app/test/event_based_scan/test_port_add.py | 7 +- app/test/event_based_scan/test_subnet_add.py | 4 +- app/test/scan/test_scan_controller.py | 9 +- 27 files changed, 699 insertions(+), 632 deletions(-) create mode 100644 app/config/link_finders.json delete mode 100644 app/discover/find_links.py delete mode 100644 app/discover/find_links_for_instance_vnics.py delete mode 100644 app/discover/find_links_for_oteps.py delete mode 100644 app/discover/find_links_for_pnics.py delete mode 100644 app/discover/find_links_for_vconnectors.py delete mode 100644 app/discover/find_links_for_vedges.py delete mode 100644 app/discover/find_links_for_vservice_vnics.py create mode 100644 app/discover/link_finders/__init__.py create mode 100644 app/discover/link_finders/find_links.py create mode 100644 app/discover/link_finders/find_links_for_instance_vnics.py create mode 100644 app/discover/link_finders/find_links_for_oteps.py create mode 100644 app/discover/link_finders/find_links_for_pnics.py create mode 100644 app/discover/link_finders/find_links_for_vconnectors.py create mode 100644 app/discover/link_finders/find_links_for_vedges.py create mode 100644 app/discover/link_finders/find_links_for_vservice_vnics.py create mode 100644 app/discover/link_finders/find_links_metadata_parser.py (limited to 'app') diff --git a/app/config/link_finders.json b/app/config/link_finders.json new file mode 100644 index 0000000..2368333 --- /dev/null +++ b/app/config/link_finders.json @@ -0,0 +1,12 @@ +{ + "finders_package": "discover.link_finders", + "base_finder": "FindLinks", + "link_finders": [ + "FindLinksForInstanceVnics", + "FindLinksForOteps", + "FindLinksForPnics", + "FindLinksForVconnectors", + "FindLinksForVedges", + "FindLinksForVserviceVnics" + ] +} \ No newline at end of file diff --git a/app/discover/events/event_interface_add.py b/app/discover/events/event_interface_add.py index 2c7af13..698559c 100644 --- a/app/discover/events/event_interface_add.py +++ b/app/discover/events/event_interface_add.py @@ -8,7 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### import time - from functools import partial from discover.events.event_base import EventBase, EventResult @@ -17,7 +16,8 @@ from discover.events.event_subnet_add import EventSubnetAdd from discover.fetchers.api.api_access import ApiAccess from discover.fetchers.api.api_fetch_regions import ApiFetchRegions from discover.fetchers.cli.cli_fetch_host_vservice import CliFetchHostVservice -from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics +from discover.link_finders.find_links_for_vservice_vnics import \ + FindLinksForVserviceVnics from discover.scanner import Scanner from utils.util import decode_router_id, encode_router_id diff --git a/app/discover/events/event_port_add.py b/app/discover/events/event_port_add.py index 63a5e80..9220015 100644 --- a/app/discover/events/event_port_add.py +++ b/app/discover/events/event_port_add.py @@ -12,10 +12,12 @@ import datetime from discover.events.event_base import EventBase, EventResult from discover.fetchers.api.api_fetch_host_instances import ApiFetchHostInstances from discover.fetchers.cli.cli_fetch_instance_vnics import CliFetchInstanceVnics -from discover.fetchers.cli.cli_fetch_instance_vnics_vpp import CliFetchInstanceVnicsVpp +from discover.fetchers.cli.cli_fetch_instance_vnics_vpp import \ + CliFetchInstanceVnicsVpp from discover.fetchers.cli.cli_fetch_vservice_vnics import CliFetchVserviceVnics -from discover.find_links_for_instance_vnics import FindLinksForInstanceVnics -from discover.find_links_for_vedges import FindLinksForVedges +from discover.link_finders.find_links_for_instance_vnics import \ + FindLinksForInstanceVnics +from discover.link_finders.find_links_for_vedges import FindLinksForVedges from discover.scanner import Scanner diff --git a/app/discover/events/event_router_add.py b/app/discover/events/event_router_add.py index 20e07e5..3c1c9e2 100644 --- a/app/discover/events/event_router_add.py +++ b/app/discover/events/event_router_add.py @@ -8,14 +8,14 @@ # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### import datetime - from functools import partial from discover.events.event_base import EventBase, EventResult from discover.events.event_port_add import EventPortAdd from discover.events.event_subnet_add import EventSubnetAdd from discover.fetchers.cli.cli_fetch_host_vservice import CliFetchHostVservice -from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics +from discover.link_finders.find_links_for_vservice_vnics import \ + FindLinksForVserviceVnics from discover.scanner import Scanner from utils.util import decode_router_id, encode_router_id diff --git a/app/discover/events/event_router_update.py b/app/discover/events/event_router_update.py index 8dd53f0..cfbbf58 100644 --- a/app/discover/events/event_router_update.py +++ b/app/discover/events/event_router_update.py @@ -11,7 +11,7 @@ from discover.events.event_base import EventBase, EventResult from discover.events.event_port_delete import EventPortDelete from discover.events.event_router_add import EventRouterAdd from discover.fetchers.cli.cli_fetch_host_vservice import CliFetchHostVservice -from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics +from discover.link_finders.find_links_for_vservice_vnics import FindLinksForVserviceVnics from discover.scanner import Scanner from utils.util import encode_router_id diff --git a/app/discover/events/event_subnet_add.py b/app/discover/events/event_subnet_add.py index a33f7cf..fcae5fd 100644 --- a/app/discover/events/event_subnet_add.py +++ b/app/discover/events/event_subnet_add.py @@ -15,8 +15,9 @@ from discover.fetchers.api.api_access import ApiAccess from discover.fetchers.api.api_fetch_port import ApiFetchPort from discover.fetchers.api.api_fetch_regions import ApiFetchRegions from discover.fetchers.db.db_fetch_port import DbFetchPort -from discover.find_links_for_pnics import FindLinksForPnics -from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics +from discover.link_finders.find_links_for_pnics import FindLinksForPnics +from discover.link_finders.find_links_for_vservice_vnics import \ + FindLinksForVserviceVnics from discover.scanner import Scanner diff --git a/app/discover/events/event_subnet_update.py b/app/discover/events/event_subnet_update.py index 26d1984..3529f0d 100644 --- a/app/discover/events/event_subnet_update.py +++ b/app/discover/events/event_subnet_update.py @@ -14,7 +14,7 @@ from discover.events.event_subnet_add import EventSubnetAdd from discover.fetchers.api.api_access import ApiAccess from discover.fetchers.api.api_fetch_regions import ApiFetchRegions from discover.fetchers.db.db_fetch_port import DbFetchPort -from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics +from discover.link_finders.find_links_for_vservice_vnics import FindLinksForVserviceVnics from discover.scanner import Scanner diff --git a/app/discover/find_links.py b/app/discover/find_links.py deleted file mode 100644 index d234479..0000000 --- a/app/discover/find_links.py +++ /dev/null @@ -1,35 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -from discover.fetcher import Fetcher -from utils.inventory_mgr import InventoryMgr - - -class FindLinks(Fetcher): - def __init__(self): - super().__init__() - self.inv = InventoryMgr() - - def create_link(self, env, source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=None, switch=None, - extra_attributes=None): - if extra_attributes is None: - extra_attributes = {} - source_label = extra_attributes.get('source_label', '') - target_label = extra_attributes.get('target_label', '') - link = self.inv.create_link(env, - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - source_label=source_label, - target_label=target_label, - host=host, switch=switch, - extra_attributes=extra_attributes) - if self.inv.monitoring_setup_manager: - self.inv.monitoring_setup_manager.create_setup(link) diff --git a/app/discover/find_links_for_instance_vnics.py b/app/discover/find_links_for_instance_vnics.py deleted file mode 100644 index 55f7bfc..0000000 --- a/app/discover/find_links_for_instance_vnics.py +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -from discover.find_links import FindLinks - - -class FindLinksForInstanceVnics(FindLinks): - def __init__(self): - super().__init__() - - def add_links(self): - self.log.info("adding links of type: instance-vnic") - vnics = self.inv.find_items({ - "environment": self.get_env(), - "type": "vnic", - "vnic_type": "instance_vnic" - }) - for v in vnics: - self.add_link_for_vnic(v) - - def add_link_for_vnic(self, v): - instance = self.inv.get_by_id(self.get_env(), v["instance_id"]) - if "network_info" not in instance: - self.log.warn("add_link_for_vnic: " + - "network_info missing in instance: %s ", - instance["id"]) - return - host = self.inv.get_by_id(self.get_env(), instance["host"]) - host_types = host["host_type"] - if "Network" not in host_types and "Compute" not in host_types: - return [] - source = instance["_id"] - source_id = instance["id"] - target = v["_id"] - target_id = v["id"] - link_type = "instance-vnic" - # find related network - network_name = None - network_id = None - for net in instance["network_info"]: - if net["devname"] == v["id"]: - network_name = net["network"]["label"] - network_id = net['network']['id'] - v['network'] = network_id - self.inv.set(v) - break - state = "up" # TBD - link_weight = 0 # TBD - attributes = {} if not network_id else {'network': network_id} - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, network_name, state, link_weight, - host=host["name"], - extra_attributes=attributes) diff --git a/app/discover/find_links_for_oteps.py b/app/discover/find_links_for_oteps.py deleted file mode 100644 index 585dc19..0000000 --- a/app/discover/find_links_for_oteps.py +++ /dev/null @@ -1,88 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -from discover.find_links import FindLinks - - -class FindLinksForOteps(FindLinks): - def __init__(self): - super().__init__() - - def add_links(self): - self.log.info("adding link types: " + - "vedge-otep, otep-vconnector, otep-host_pnic") - oteps = self.inv.find_items({ - "environment": self.get_env(), - "type": "otep" - }) - for otep in oteps: - self.add_vedge_otep_link(otep) - self.add_otep_vconnector_link(otep) - self.add_otep_pnic_link(otep) - - def add_vedge_otep_link(self, otep): - vedge = self.inv.get_by_id(self.get_env(), otep["parent_id"]) - source = vedge["_id"] - source_id = vedge["id"] - target = otep["_id"] - target_id = otep["id"] - link_type = "vedge-otep" - link_name = vedge["name"] + "-otep" - state = "up" # TBD - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=vedge["host"]) - - def add_otep_vconnector_link(self, otep): - if "vconnector" not in otep: - return - vconnector = self.inv.find_items({ - "environment": self.get_env(), - "type": "vconnector", - "host": otep["host"], - "name": otep["vconnector"] - }, get_single=True) - if not vconnector: - return - source = otep["_id"] - source_id = otep["id"] - target = vconnector["_id"] - target_id = vconnector["id"] - link_type = "otep-vconnector" - link_name = otep["name"] + "-" + otep["vconnector"] - state = "up" # TBD - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=otep["host"]) - - def add_otep_pnic_link(self, otep): - pnic = self.inv.find_items({ - "environment": self.get_env(), - "type": "host_pnic", - "host": otep["host"], - "IP Address": otep["ip_address"] - }, get_single=True) - if not pnic: - return - source = otep["_id"] - source_id = otep["id"] - target = pnic["_id"] - target_id = pnic["id"] - link_type = "otep-host_pnic" - link_name = otep["host"] + "pnic" + pnic["name"] - state = "up" # TBD - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=otep["host"]) diff --git a/app/discover/find_links_for_pnics.py b/app/discover/find_links_for_pnics.py deleted file mode 100644 index 98d481c..0000000 --- a/app/discover/find_links_for_pnics.py +++ /dev/null @@ -1,132 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -import re - -from discover.find_links import FindLinks -from utils.util import decode_aci_dn - - -class FindLinksForPnics(FindLinks): - def __init__(self): - super().__init__() - - def add_links(self): - self.log.info("adding links of type: pnic-network, " - "host_pnic-switch_pnic, switch-host_pnic") - pnics = self.inv.find_items({ - "environment": self.get_env(), - "type": "host_pnic" - }) - for pnic in pnics: - self.add_pnic_network_links(pnic) - self.add_host_pnic_to_switch_pnic_link(pnic) - - self.log.info("adding links of type: switch_pnic-switch_pnic, " - "switch-switch_pnic") - pnics = self.inv.find_items({ - "environment": self.get_env(), - "type": "switch_pnic", - }) - for pnic in pnics: - self.add_switch_to_pnic_link(pnic) - if pnic["role"] == "uplink": - self.add_switch_pnic_to_switch_pnic_link(pnic) - - def add_pnic_network_links(self, pnic): - host = pnic["host"] - # find ports for that host, and fetch just the network ID - ports = self.inv.find_items({ - "environment": self.get_env(), - "type": "port", - "binding:host_id": host - }, {"network_id": 1, "id": 1}) - networks = {} - for port in ports: - networks[port["network_id"]] = 1 - for network_id in networks.keys(): - network = self.inv.get_by_id(self.get_env(), network_id) - if not network: - return - source = pnic["_id"] - source_id = pnic["id"] - target = network["_id"] - target_id = network["id"] - link_type = "host_pnic-network" - link_name = "Segment-" + str(network["provider:segmentation_id"]) \ - if "provider:segmentation_id" in network \ - else "Segment-None" - state = "up" if pnic["Link detected"] == "yes" else "down" - link_weight = 0 # TBD - attributes = {"network": target_id} - if "port_id" in pnic: - attributes['source_label'] = "port-" + pnic["port_id"] - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=host, - extra_attributes=attributes) - - def add_host_pnic_to_switch_pnic_link(self, host_pnic): - switch_pnic = self.inv.find_items({ - "environment": self.get_env(), - "type": "switch_pnic", - "mac_address": host_pnic["mac_address"]}, - get_single=True) - if not switch_pnic: - return - source = host_pnic["_id"] - source_id = host_pnic["id"] - target = switch_pnic["_id"] - target_id = switch_pnic["id"] - link_type = "host_pnic-switch_pnic" - link_name = "{}-{}".format(host_pnic['host'], - switch_pnic['parent_id']) - state = "up" if host_pnic["Link detected"] == "yes" else "down" - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=host_pnic['host']) - - def add_switch_pnic_to_switch_pnic_link(self, leaf_pnic): - spine_pnic = self.inv.get_by_id(self.get_env(), - leaf_pnic['connected_to']) - if not spine_pnic: - return - source = leaf_pnic["_id"] - source_id = leaf_pnic["id"] - target = spine_pnic["_id"] - target_id = spine_pnic["id"] - link_type = "switch_pnic-switch_pnic" - if_id_matches = re.search("(eth.*)$", source_id) - link_name = decode_aci_dn(if_id_matches.group(1)) - state = "up" # TBD - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - switch=leaf_pnic['switch']) - - def add_switch_to_pnic_link(self, pnic): - switch = self.inv.get_by_id(self.get_env(), pnic['parent_id']) - if not switch: - return - source = switch["_id"] - source_id = switch["id"] - target = pnic["_id"] - target_id = pnic["id"] - link_type = "switch-{}".format(pnic['type']) - link_name = "{}={}".format(switch["object_name"], pnic["object_name"]) - state = "up" # TBD - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - switch=switch['id']) diff --git a/app/discover/find_links_for_vconnectors.py b/app/discover/find_links_for_vconnectors.py deleted file mode 100644 index ab96ebe..0000000 --- a/app/discover/find_links_for_vconnectors.py +++ /dev/null @@ -1,91 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -from discover.find_links import FindLinks - - -class FindLinksForVconnectors(FindLinks): - def __init__(self): - super().__init__() - - def add_links(self): - vconnectors = self.inv.find_items({ - "environment": self.get_env(), - "type": "vconnector" - }) - self.log.info("adding links of type: vnic-vconnector, " - "vconnector-host_pnic") - for vconnector in vconnectors: - for interface in vconnector["interfaces_names"]: - self.add_vnic_vconnector_link(vconnector, interface) - self.add_vconnector_pnic_link(vconnector, interface) - - def add_vnic_vconnector_link(self, vconnector, interface_name): - mechanism_drivers = self.configuration.environment['mechanism_drivers'] - is_ovs = mechanism_drivers and mechanism_drivers[0] == 'OVS' - if is_ovs: - # interface ID for OVS - vnic = self.inv.get_by_id(self.get_env(), interface_name) - else: - # interface ID for VPP - match interface MAC address to vNIC MAC - interface = vconnector['interfaces'][interface_name] - if not interface or 'mac_address' not in interface: - return - vnic_mac = interface['mac_address'] - vnic = self.inv.get_by_field(self.get_env(), 'vnic', - 'mac_address', vnic_mac, - get_single=True) - if not vnic: - return - host = vnic["host"] - source = vnic["_id"] - source_id = vnic["id"] - target = vconnector["_id"] - target_id = vconnector["id"] - link_type = "vnic-vconnector" - link_name = vnic["mac_address"] - state = "up" # TBD - link_weight = 0 # TBD - attributes = {} - if 'network' in vnic: - attributes = {'network': vnic['network']} - vconnector['network'] = vnic['network'] - self.inv.set(vconnector) - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=host, - extra_attributes=attributes) - - def add_vconnector_pnic_link(self, vconnector, interface): - ifname = interface['name'] if isinstance(interface, dict) else interface - if "." in ifname: - ifname = ifname[:ifname.index(".")] - host = vconnector["host"] - pnic = self.inv.find_items({ - "environment": self.get_env(), - "type": "host_pnic", - "host": vconnector["host"], - "name": ifname - }, get_single=True) - if not pnic: - return - source = vconnector["_id"] - source_id = vconnector["id"] - target = pnic["_id"] - target_id = pnic["id"] - link_type = "vconnector-host_pnic" - link_name = pnic["name"] - state = "up" # TBD - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, - target, target_id, - link_type, link_name, state, link_weight, - host=host) diff --git a/app/discover/find_links_for_vedges.py b/app/discover/find_links_for_vedges.py deleted file mode 100644 index acafceb..0000000 --- a/app/discover/find_links_for_vedges.py +++ /dev/null @@ -1,128 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -from discover.find_links import FindLinks - - -class FindLinksForVedges(FindLinks): - def __init__(self): - super().__init__() - - def add_links(self): - self.log.info("adding link types: " + - "vnic-vedge, vconnector-vedge, vedge-host_pnic") - vedges = self.inv.find_items({ - "environment": self.get_env(), - "type": "vedge" - }) - for vedge in vedges: - ports = vedge["ports"] - for p in ports.values(): - self.add_link_for_vedge(vedge, p) - - def add_link_for_vedge(self, vedge, port): - vnic = self.inv.get_by_id(self.get_env(), - vedge['host'] + '-' + port["name"]) - if not vnic: - self.find_matching_vconnector(vedge, port) - self.find_matching_pnic(vedge, port) - return - source = vnic["_id"] - source_id = vnic["id"] - target = vedge["_id"] - target_id = vedge["id"] - link_type = "vnic-vedge" - link_name = vnic["name"] + "-" + vedge["name"] - if "tag" in port: - link_name += "-" + port["tag"] - state = "up" # TBD - link_weight = 0 # TBD - source_label = vnic["mac_address"] - target_label = port["id"] - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=vedge["host"], - extra_attributes={"source_label": source_label, - "target_label": target_label}) - - def find_matching_vconnector(self, vedge, port): - if self.configuration.has_network_plugin('VPP'): - vconnector_interface_name = port['name'] - else: - if not port["name"].startswith("qv"): - return - base_id = port["name"][3:] - vconnector_interface_name = "qvb" + base_id - vconnector = self.inv.find_items({ - "environment": self.get_env(), - "type": "vconnector", - "host": vedge['host'], - 'interfaces_names': vconnector_interface_name}, - get_single=True) - if not vconnector: - return - source = vconnector["_id"] - source_id = vconnector["id"] - target = vedge["_id"] - target_id = vedge["id"] - link_type = "vconnector-vedge" - link_name = "port-" + port["id"] - if "tag" in port: - link_name += "-" + port["tag"] - state = "up" # TBD - link_weight = 0 # TBD - source_label = vconnector_interface_name - target_label = port["name"] - mac_address = "Unknown" - attributes = {'mac_address': mac_address, 'source_label': source_label, - 'target_label': target_label} - for interface in vconnector['interfaces'].values(): - if vconnector_interface_name != interface['name']: - continue - if 'mac_address' not in interface: - continue - mac_address = interface['mac_address'] - attributes['mac_address'] = mac_address - break - if 'network' in vconnector: - attributes['network'] = vconnector['network'] - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=vedge["host"], - extra_attributes=attributes) - - def find_matching_pnic(self, vedge, port): - pname = port["name"] - if "pnic" in vedge: - if pname != vedge["pnic"]: - return - elif self.configuration.has_network_plugin('VPP'): - pass - pnic = self.inv.find_items({ - "environment": self.get_env(), - "type": "host_pnic", - "host": vedge["host"], - "name": pname - }, get_single=True) - if not pnic: - return - source = vedge["_id"] - source_id = vedge["id"] - target = pnic["_id"] - target_id = pnic["id"] - link_type = "vedge-host_pnic" - link_name = "Port-" + port["id"] - state = "up" if pnic["Link detected"] == "yes" else "down" - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, target, target_id, - link_type, link_name, state, link_weight, - host=vedge["host"]) diff --git a/app/discover/find_links_for_vservice_vnics.py b/app/discover/find_links_for_vservice_vnics.py deleted file mode 100644 index 3b8f32d..0000000 --- a/app/discover/find_links_for_vservice_vnics.py +++ /dev/null @@ -1,57 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -from discover.find_links import FindLinks - - -class FindLinksForVserviceVnics(FindLinks): - def __init__(self): - super().__init__() - - def add_links(self, search=None): - self.log.info("adding links of type: vservice-vnic") - - if search is None: - search = {} - - search.update({"environment": self.get_env(), - "type": "vnic", - "vnic_type": "vservice_vnic"}) - - vnics = self.inv.find_items(search) - - for v in vnics: - self.add_link_for_vnic(v) - - def add_link_for_vnic(self, v): - host = self.inv.get_by_id(self.get_env(), v["host"]) - if "Network" not in host["host_type"]: - return - if "network" not in v: - return - network = self.inv.get_by_id(self.get_env(), v["network"]) - if network == []: - return - vservice_id = v["parent_id"] - vservice_id = vservice_id[:vservice_id.rindex('-')] - vservice = self.inv.get_by_id(self.get_env(), vservice_id) - source = vservice["_id"] - source_id = vservice_id - target = v["_id"] - target_id = v["id"] - link_type = "vservice-vnic" - link_name = network["name"] - state = "up" # TBD - link_weight = 0 # TBD - self.create_link(self.get_env(), - source, source_id, - target, target_id, - link_type, link_name, state, link_weight, - host=v["host"], - extra_attributes={'network': v['network']}) diff --git a/app/discover/link_finders/__init__.py b/app/discover/link_finders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/discover/link_finders/find_links.py b/app/discover/link_finders/find_links.py new file mode 100644 index 0000000..d234479 --- /dev/null +++ b/app/discover/link_finders/find_links.py @@ -0,0 +1,35 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetcher import Fetcher +from utils.inventory_mgr import InventoryMgr + + +class FindLinks(Fetcher): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def create_link(self, env, source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=None, switch=None, + extra_attributes=None): + if extra_attributes is None: + extra_attributes = {} + source_label = extra_attributes.get('source_label', '') + target_label = extra_attributes.get('target_label', '') + link = self.inv.create_link(env, + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + source_label=source_label, + target_label=target_label, + host=host, switch=switch, + extra_attributes=extra_attributes) + if self.inv.monitoring_setup_manager: + self.inv.monitoring_setup_manager.create_setup(link) diff --git a/app/discover/link_finders/find_links_for_instance_vnics.py b/app/discover/link_finders/find_links_for_instance_vnics.py new file mode 100644 index 0000000..7e0273d --- /dev/null +++ b/app/discover/link_finders/find_links_for_instance_vnics.py @@ -0,0 +1,60 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForInstanceVnics(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + self.log.info("adding links of type: instance-vnic") + vnics = self.inv.find_items({ + "environment": self.get_env(), + "type": "vnic", + "vnic_type": "instance_vnic" + }) + for v in vnics: + self.add_link_for_vnic(v) + + def add_link_for_vnic(self, v): + instance = self.inv.get_by_id(self.get_env(), v["instance_id"]) + if "network_info" not in instance: + self.log.warn("add_link_for_vnic: " + + "network_info missing in instance: %s ", + instance["id"]) + return + host = self.inv.get_by_id(self.get_env(), instance["host"]) + host_types = host["host_type"] + if "Network" not in host_types and "Compute" not in host_types: + return [] + source = instance["_id"] + source_id = instance["id"] + target = v["_id"] + target_id = v["id"] + link_type = "instance-vnic" + # find related network + network_name = None + network_id = None + for net in instance["network_info"]: + if net["devname"] == v["id"]: + network_name = net["network"]["label"] + network_id = net['network']['id'] + v['network'] = network_id + self.inv.set(v) + break + state = "up" # TBD + link_weight = 0 # TBD + attributes = {} if not network_id else {'network': network_id} + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, network_name, state, link_weight, + host=host["name"], + extra_attributes=attributes) diff --git a/app/discover/link_finders/find_links_for_oteps.py b/app/discover/link_finders/find_links_for_oteps.py new file mode 100644 index 0000000..b5d1667 --- /dev/null +++ b/app/discover/link_finders/find_links_for_oteps.py @@ -0,0 +1,88 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForOteps(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + self.log.info("adding link types: " + + "vedge-otep, otep-vconnector, otep-host_pnic") + oteps = self.inv.find_items({ + "environment": self.get_env(), + "type": "otep" + }) + for otep in oteps: + self.add_vedge_otep_link(otep) + self.add_otep_vconnector_link(otep) + self.add_otep_pnic_link(otep) + + def add_vedge_otep_link(self, otep): + vedge = self.inv.get_by_id(self.get_env(), otep["parent_id"]) + source = vedge["_id"] + source_id = vedge["id"] + target = otep["_id"] + target_id = otep["id"] + link_type = "vedge-otep" + link_name = vedge["name"] + "-otep" + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=vedge["host"]) + + def add_otep_vconnector_link(self, otep): + if "vconnector" not in otep: + return + vconnector = self.inv.find_items({ + "environment": self.get_env(), + "type": "vconnector", + "host": otep["host"], + "name": otep["vconnector"] + }, get_single=True) + if not vconnector: + return + source = otep["_id"] + source_id = otep["id"] + target = vconnector["_id"] + target_id = vconnector["id"] + link_type = "otep-vconnector" + link_name = otep["name"] + "-" + otep["vconnector"] + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=otep["host"]) + + def add_otep_pnic_link(self, otep): + pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "host_pnic", + "host": otep["host"], + "IP Address": otep["ip_address"] + }, get_single=True) + if not pnic: + return + source = otep["_id"] + source_id = otep["id"] + target = pnic["_id"] + target_id = pnic["id"] + link_type = "otep-host_pnic" + link_name = otep["host"] + "pnic" + pnic["name"] + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=otep["host"]) diff --git a/app/discover/link_finders/find_links_for_pnics.py b/app/discover/link_finders/find_links_for_pnics.py new file mode 100644 index 0000000..1f02426 --- /dev/null +++ b/app/discover/link_finders/find_links_for_pnics.py @@ -0,0 +1,132 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.link_finders.find_links import FindLinks +from utils.util import decode_aci_dn + + +class FindLinksForPnics(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + self.log.info("adding links of type: pnic-network, " + "host_pnic-switch_pnic, switch-host_pnic") + pnics = self.inv.find_items({ + "environment": self.get_env(), + "type": "host_pnic" + }) + for pnic in pnics: + self.add_pnic_network_links(pnic) + self.add_host_pnic_to_switch_pnic_link(pnic) + + self.log.info("adding links of type: switch_pnic-switch_pnic, " + "switch-switch_pnic") + pnics = self.inv.find_items({ + "environment": self.get_env(), + "type": "switch_pnic", + }) + for pnic in pnics: + self.add_switch_to_pnic_link(pnic) + if pnic["role"] == "uplink": + self.add_switch_pnic_to_switch_pnic_link(pnic) + + def add_pnic_network_links(self, pnic): + host = pnic["host"] + # find ports for that host, and fetch just the network ID + ports = self.inv.find_items({ + "environment": self.get_env(), + "type": "port", + "binding:host_id": host + }, {"network_id": 1, "id": 1}) + networks = {} + for port in ports: + networks[port["network_id"]] = 1 + for network_id in networks.keys(): + network = self.inv.get_by_id(self.get_env(), network_id) + if not network: + return + source = pnic["_id"] + source_id = pnic["id"] + target = network["_id"] + target_id = network["id"] + link_type = "host_pnic-network" + link_name = "Segment-" + str(network["provider:segmentation_id"]) \ + if "provider:segmentation_id" in network \ + else "Segment-None" + state = "up" if pnic["Link detected"] == "yes" else "down" + link_weight = 0 # TBD + attributes = {"network": target_id} + if "port_id" in pnic: + attributes['source_label'] = "port-" + pnic["port_id"] + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=host, + extra_attributes=attributes) + + def add_host_pnic_to_switch_pnic_link(self, host_pnic): + switch_pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "switch_pnic", + "mac_address": host_pnic["mac_address"]}, + get_single=True) + if not switch_pnic: + return + source = host_pnic["_id"] + source_id = host_pnic["id"] + target = switch_pnic["_id"] + target_id = switch_pnic["id"] + link_type = "host_pnic-switch_pnic" + link_name = "{}-{}".format(host_pnic['host'], + switch_pnic['parent_id']) + state = "up" if host_pnic["Link detected"] == "yes" else "down" + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=host_pnic['host']) + + def add_switch_pnic_to_switch_pnic_link(self, leaf_pnic): + spine_pnic = self.inv.get_by_id(self.get_env(), + leaf_pnic['connected_to']) + if not spine_pnic: + return + source = leaf_pnic["_id"] + source_id = leaf_pnic["id"] + target = spine_pnic["_id"] + target_id = spine_pnic["id"] + link_type = "switch_pnic-switch_pnic" + if_id_matches = re.search("(eth.*)$", source_id) + link_name = decode_aci_dn(if_id_matches.group(1)) + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + switch=leaf_pnic['switch']) + + def add_switch_to_pnic_link(self, pnic): + switch = self.inv.get_by_id(self.get_env(), pnic['parent_id']) + if not switch: + return + source = switch["_id"] + source_id = switch["id"] + target = pnic["_id"] + target_id = pnic["id"] + link_type = "switch-{}".format(pnic['type']) + link_name = "{}={}".format(switch["object_name"], pnic["object_name"]) + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + switch=switch['id']) diff --git a/app/discover/link_finders/find_links_for_vconnectors.py b/app/discover/link_finders/find_links_for_vconnectors.py new file mode 100644 index 0000000..edb351a --- /dev/null +++ b/app/discover/link_finders/find_links_for_vconnectors.py @@ -0,0 +1,91 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForVconnectors(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + vconnectors = self.inv.find_items({ + "environment": self.get_env(), + "type": "vconnector" + }) + self.log.info("adding links of type: vnic-vconnector, " + "vconnector-host_pnic") + for vconnector in vconnectors: + for interface in vconnector["interfaces_names"]: + self.add_vnic_vconnector_link(vconnector, interface) + self.add_vconnector_pnic_link(vconnector, interface) + + def add_vnic_vconnector_link(self, vconnector, interface_name): + mechanism_drivers = self.configuration.environment['mechanism_drivers'] + is_ovs = mechanism_drivers and mechanism_drivers[0] == 'OVS' + if is_ovs: + # interface ID for OVS + vnic = self.inv.get_by_id(self.get_env(), interface_name) + else: + # interface ID for VPP - match interface MAC address to vNIC MAC + interface = vconnector['interfaces'][interface_name] + if not interface or 'mac_address' not in interface: + return + vnic_mac = interface['mac_address'] + vnic = self.inv.get_by_field(self.get_env(), 'vnic', + 'mac_address', vnic_mac, + get_single=True) + if not vnic: + return + host = vnic["host"] + source = vnic["_id"] + source_id = vnic["id"] + target = vconnector["_id"] + target_id = vconnector["id"] + link_type = "vnic-vconnector" + link_name = vnic["mac_address"] + state = "up" # TBD + link_weight = 0 # TBD + attributes = {} + if 'network' in vnic: + attributes = {'network': vnic['network']} + vconnector['network'] = vnic['network'] + self.inv.set(vconnector) + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=host, + extra_attributes=attributes) + + def add_vconnector_pnic_link(self, vconnector, interface): + ifname = interface['name'] if isinstance(interface, dict) else interface + if "." in ifname: + ifname = ifname[:ifname.index(".")] + host = vconnector["host"] + pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "host_pnic", + "host": vconnector["host"], + "name": ifname + }, get_single=True) + if not pnic: + return + source = vconnector["_id"] + source_id = vconnector["id"] + target = pnic["_id"] + target_id = pnic["id"] + link_type = "vconnector-host_pnic" + link_name = pnic["name"] + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, + target, target_id, + link_type, link_name, state, link_weight, + host=host) diff --git a/app/discover/link_finders/find_links_for_vedges.py b/app/discover/link_finders/find_links_for_vedges.py new file mode 100644 index 0000000..f9719b4 --- /dev/null +++ b/app/discover/link_finders/find_links_for_vedges.py @@ -0,0 +1,128 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForVedges(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + self.log.info("adding link types: " + + "vnic-vedge, vconnector-vedge, vedge-host_pnic") + vedges = self.inv.find_items({ + "environment": self.get_env(), + "type": "vedge" + }) + for vedge in vedges: + ports = vedge["ports"] + for p in ports.values(): + self.add_link_for_vedge(vedge, p) + + def add_link_for_vedge(self, vedge, port): + vnic = self.inv.get_by_id(self.get_env(), + vedge['host'] + '-' + port["name"]) + if not vnic: + self.find_matching_vconnector(vedge, port) + self.find_matching_pnic(vedge, port) + return + source = vnic["_id"] + source_id = vnic["id"] + target = vedge["_id"] + target_id = vedge["id"] + link_type = "vnic-vedge" + link_name = vnic["name"] + "-" + vedge["name"] + if "tag" in port: + link_name += "-" + port["tag"] + state = "up" # TBD + link_weight = 0 # TBD + source_label = vnic["mac_address"] + target_label = port["id"] + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=vedge["host"], + extra_attributes={"source_label": source_label, + "target_label": target_label}) + + def find_matching_vconnector(self, vedge, port): + if self.configuration.has_network_plugin('VPP'): + vconnector_interface_name = port['name'] + else: + if not port["name"].startswith("qv"): + return + base_id = port["name"][3:] + vconnector_interface_name = "qvb" + base_id + vconnector = self.inv.find_items({ + "environment": self.get_env(), + "type": "vconnector", + "host": vedge['host'], + 'interfaces_names': vconnector_interface_name}, + get_single=True) + if not vconnector: + return + source = vconnector["_id"] + source_id = vconnector["id"] + target = vedge["_id"] + target_id = vedge["id"] + link_type = "vconnector-vedge" + link_name = "port-" + port["id"] + if "tag" in port: + link_name += "-" + port["tag"] + state = "up" # TBD + link_weight = 0 # TBD + source_label = vconnector_interface_name + target_label = port["name"] + mac_address = "Unknown" + attributes = {'mac_address': mac_address, 'source_label': source_label, + 'target_label': target_label} + for interface in vconnector['interfaces'].values(): + if vconnector_interface_name != interface['name']: + continue + if 'mac_address' not in interface: + continue + mac_address = interface['mac_address'] + attributes['mac_address'] = mac_address + break + if 'network' in vconnector: + attributes['network'] = vconnector['network'] + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=vedge["host"], + extra_attributes=attributes) + + def find_matching_pnic(self, vedge, port): + pname = port["name"] + if "pnic" in vedge: + if pname != vedge["pnic"]: + return + elif self.configuration.has_network_plugin('VPP'): + pass + pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "host_pnic", + "host": vedge["host"], + "name": pname + }, get_single=True) + if not pnic: + return + source = vedge["_id"] + source_id = vedge["id"] + target = pnic["_id"] + target_id = pnic["id"] + link_type = "vedge-host_pnic" + link_name = "Port-" + port["id"] + state = "up" if pnic["Link detected"] == "yes" else "down" + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=vedge["host"]) diff --git a/app/discover/link_finders/find_links_for_vservice_vnics.py b/app/discover/link_finders/find_links_for_vservice_vnics.py new file mode 100644 index 0000000..ca9bc4a --- /dev/null +++ b/app/discover/link_finders/find_links_for_vservice_vnics.py @@ -0,0 +1,57 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForVserviceVnics(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self, search=None): + self.log.info("adding links of type: vservice-vnic") + + if search is None: + search = {} + + search.update({"environment": self.get_env(), + "type": "vnic", + "vnic_type": "vservice_vnic"}) + + vnics = self.inv.find_items(search) + + for v in vnics: + self.add_link_for_vnic(v) + + def add_link_for_vnic(self, v): + host = self.inv.get_by_id(self.get_env(), v["host"]) + if "Network" not in host["host_type"]: + return + if "network" not in v: + return + network = self.inv.get_by_id(self.get_env(), v["network"]) + if network == []: + return + vservice_id = v["parent_id"] + vservice_id = vservice_id[:vservice_id.rindex('-')] + vservice = self.inv.get_by_id(self.get_env(), vservice_id) + source = vservice["_id"] + source_id = vservice_id + target = v["_id"] + target_id = v["id"] + link_type = "vservice-vnic" + link_name = network["name"] + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, + target, target_id, + link_type, link_name, state, link_weight, + host=v["host"], + extra_attributes={'network': v['network']}) diff --git a/app/discover/link_finders/find_links_metadata_parser.py b/app/discover/link_finders/find_links_metadata_parser.py new file mode 100644 index 0000000..1f28262 --- /dev/null +++ b/app/discover/link_finders/find_links_metadata_parser.py @@ -0,0 +1,57 @@ +from utils.metadata_parser import MetadataParser +from utils.util import ClassResolver + + +class FindLinksMetadataParser(MetadataParser): + + FINDERS_FILE = "link_finders.json" + + FINDERS_PACKAGE = "finders_package" + BASE_FINDER = "base_finder" + LINK_FINDERS = "link_finders" + + def __init__(self): + super().__init__() + self.finders_package = None + self.base_finder = None + self.link_finders = [] + + def validate_link_finder(self, finder_class): + try: + module_name = ClassResolver.get_module_file_by_class_name(finder_class) + instance = ClassResolver\ + .get_instance_of_class(package_name=self.finders_package, + module_name=module_name, + class_name=finder_class) + except ValueError: + instance = None + + if instance: + self.link_finders.append(instance) + else: + self.add_error('Failed to import link finder class "{}"' + .format(finder_class)) + + def validate_metadata(self, metadata: dict): + super().validate_metadata(metadata) + self.finders_package = metadata[self.FINDERS_PACKAGE] + self.base_finder = metadata[self.BASE_FINDER] + base_finder_module = ClassResolver\ + .get_module_file_by_class_name(self.base_finder) + + base_finder_class = ClassResolver.get_class_name_by_module( + ".".join((self.finders_package, base_finder_module))) + + if not base_finder_class: + self.add_error("Couldn't find base link finder class") + return + + for link_finder in metadata[self.LINK_FINDERS]: + self.validate_link_finder(finder_class=link_finder) + metadata[self.LINK_FINDERS] = self.link_finders + + return len(self.errors) == 0 + + def get_required_fields(self) -> list: + return [self.FINDERS_PACKAGE, self.BASE_FINDER, self.LINK_FINDERS] + diff --git a/app/discover/scanner.py b/app/discover/scanner.py index c310ae7..3a0b2a5 100644 --- a/app/discover/scanner.py +++ b/app/discover/scanner.py @@ -10,19 +10,15 @@ # base class for scanners import json -import queue import os +import queue import traceback from discover.clique_finder import CliqueFinder from discover.configuration import Configuration from discover.fetcher import Fetcher -from discover.find_links_for_instance_vnics import FindLinksForInstanceVnics -from discover.find_links_for_oteps import FindLinksForOteps -from discover.find_links_for_pnics import FindLinksForPnics -from discover.find_links_for_vconnectors import FindLinksForVconnectors -from discover.find_links_for_vedges import FindLinksForVedges -from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics +from discover.link_finders.find_links_metadata_parser import \ + FindLinksMetadataParser from discover.scan_error import ScanError from discover.scan_metadata_parser import ScanMetadataParser from utils.inventory_mgr import InventoryMgr @@ -49,7 +45,9 @@ class Scanner(Fetcher): self.inv = InventoryMgr() self.scanners_package = None self.scanners = {} - self.load_metadata() + self.link_finders = [] + self.load_scanners_metadata() + self.load_link_finders_metadata() def scan(self, scanner_type, obj, id_field="id", limit_to_child_id=None, limit_to_child_type=None): @@ -223,15 +221,7 @@ class Scanner(Fetcher): def scan_links(self): self.log.info("scanning for links") - fetchers_implementing_add_links = [ - FindLinksForPnics(), - FindLinksForInstanceVnics(), - FindLinksForVserviceVnics(), - FindLinksForVconnectors(), - FindLinksForVedges(), - FindLinksForOteps() - ] - for fetcher in fetchers_implementing_add_links: + for fetcher in self.link_finders: fetcher.set_env(self.get_env()) fetcher.add_links() @@ -245,7 +235,7 @@ class Scanner(Fetcher): if not ret: self.found_errors[self.get_env()] = True - def load_metadata(self): + def load_scanners_metadata(self): parser = ScanMetadataParser(self.inv) conf = self.config.get_env_config() scanners_file = os.path.join(conf.get('app_path', '/etc/calipso'), @@ -256,6 +246,15 @@ class Scanner(Fetcher): self.scanners_package = metadata[ScanMetadataParser.SCANNERS_PACKAGE] self.scanners = metadata[ScanMetadataParser.SCANNERS] + def load_link_finders_metadata(self): + parser = FindLinksMetadataParser() + conf = self.config.get_env_config() + finders_file = os.path.join(conf.get('app_path', '/etc/calipso'), + 'config', + FindLinksMetadataParser.FINDERS_FILE) + metadata = parser.parse_metadata_file(finders_file) + self.link_finders = metadata[FindLinksMetadataParser.LINK_FINDERS] + def get_scanner_package(self): return self.scanners_package diff --git a/app/test/event_based_scan/test_port_add.py b/app/test/event_based_scan/test_port_add.py index 58926a9..caa8401 100644 --- a/app/test/event_based_scan/test_port_add.py +++ b/app/test/event_based_scan/test_port_add.py @@ -7,14 +7,9 @@ # which accompanies this distribution, and is available at # # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### -from unittest.mock import MagicMock, patch +from unittest.mock import patch from discover.events.event_port_add import EventPortAdd -from discover.fetchers.api.api_fetch_host_instances import ApiFetchHostInstances -from discover.fetchers.cli.cli_fetch_instance_vnics import CliFetchInstanceVnics -from discover.find_links_for_instance_vnics import FindLinksForInstanceVnics -from discover.find_links_for_vedges import FindLinksForVedges -from discover.scanner import Scanner from test.event_based_scan.test_data.event_payload_port_add import \ EVENT_PAYLOAD_PORT_INSTANCE_ADD, NETWORK_DOC, \ INSTANCE_DOC, INSTANCES_ROOT, VNIC_DOCS, INSTANCE_DOCS, PORTS_FOLDER, \ diff --git a/app/test/event_based_scan/test_subnet_add.py b/app/test/event_based_scan/test_subnet_add.py index ce934f3..b9145a5 100644 --- a/app/test/event_based_scan/test_subnet_add.py +++ b/app/test/event_based_scan/test_subnet_add.py @@ -7,12 +7,10 @@ # which accompanies this distribution, and is available at # # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### -from unittest.mock import MagicMock, patch +from unittest.mock import patch from discover.events.event_subnet_add import EventSubnetAdd from discover.fetchers.api.api_access import ApiAccess -from discover.find_links_for_pnics import FindLinksForPnics -from discover.find_links_for_vservice_vnics import FindLinksForVserviceVnics from test.event_based_scan.test_data.event_payload_subnet_add import \ EVENT_PAYLOAD_SUBNET_ADD, \ EVENT_PAYLOAD_REGION, NETWORK_DOC, HOST_DOC, PORT_DOC diff --git a/app/test/scan/test_scan_controller.py b/app/test/scan/test_scan_controller.py index f3bcc9a..f903e11 100644 --- a/app/test/scan/test_scan_controller.py +++ b/app/test/scan/test_scan_controller.py @@ -140,20 +140,23 @@ class TestScanController(TestScan): deploy_monitoring_setup_count) def prepare_scan_mocks(self): - self.load_metadata = Scanner.load_metadata + self.load_link_finders_metadata = Scanner.load_scanners_metadata + self.load_scanners_metadata = Scanner.load_scanners_metadata self.scan = Scanner.scan self.scan_links = Scanner.scan_links self.scan_cliques = Scanner.scan_cliques self.deploy_monitoring_setup = Scanner.deploy_monitoring_setup - Scanner.load_metadata = MagicMock() + Scanner.load_link_finders_metadata = MagicMock() + Scanner.load_scanners_metadata = MagicMock() Scanner.scan = MagicMock() Scanner.scan_links = MagicMock() Scanner.scan_cliques = MagicMock() Scanner.deploy_monitoring_setup = MagicMock() def reset_methods(self): - Scanner.load_metadata = self.load_metadata + Scanner.load_link_finders_metadata = self.load_link_finders_metadata + Scanner.load_scanners_metadata = self.load_scanners_metadata Scanner.scan = self.scan Scanner.scan_links = self.scan_links Scanner.scan_cliques = self.scan_cliques -- cgit 1.2.3-korg