From 7b4235dced0c66096638bebd5cc98a631538b0e1 Mon Sep 17 00:00:00 2001 From: Ilia Abashin Date: Fri, 1 Sep 2017 15:51:52 +0300 Subject: Refactored link finders Mappings are now defined in a configuration file and fetched dynamically. Change-Id: I250c22967fc66fc0aca173d4c9d65581d879b5d2 Signed-off-by: Ilia Abashin --- app/discover/link_finders/__init__.py | 0 app/discover/link_finders/find_links.py | 35 ++++++ .../link_finders/find_links_for_instance_vnics.py | 60 ++++++++++ app/discover/link_finders/find_links_for_oteps.py | 88 ++++++++++++++ app/discover/link_finders/find_links_for_pnics.py | 132 +++++++++++++++++++++ .../link_finders/find_links_for_vconnectors.py | 91 ++++++++++++++ app/discover/link_finders/find_links_for_vedges.py | 128 ++++++++++++++++++++ .../link_finders/find_links_for_vservice_vnics.py | 57 +++++++++ .../link_finders/find_links_metadata_parser.py | 57 +++++++++ 9 files changed, 648 insertions(+) create mode 100644 app/discover/link_finders/__init__.py create mode 100644 app/discover/link_finders/find_links.py create mode 100644 app/discover/link_finders/find_links_for_instance_vnics.py create mode 100644 app/discover/link_finders/find_links_for_oteps.py create mode 100644 app/discover/link_finders/find_links_for_pnics.py create mode 100644 app/discover/link_finders/find_links_for_vconnectors.py create mode 100644 app/discover/link_finders/find_links_for_vedges.py create mode 100644 app/discover/link_finders/find_links_for_vservice_vnics.py create mode 100644 app/discover/link_finders/find_links_metadata_parser.py (limited to 'app/discover/link_finders') diff --git a/app/discover/link_finders/__init__.py b/app/discover/link_finders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/discover/link_finders/find_links.py b/app/discover/link_finders/find_links.py new file mode 100644 index 0000000..d234479 --- /dev/null +++ b/app/discover/link_finders/find_links.py @@ -0,0 +1,35 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetcher import Fetcher +from utils.inventory_mgr import InventoryMgr + + +class FindLinks(Fetcher): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def create_link(self, env, source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=None, switch=None, + extra_attributes=None): + if extra_attributes is None: + extra_attributes = {} + source_label = extra_attributes.get('source_label', '') + target_label = extra_attributes.get('target_label', '') + link = self.inv.create_link(env, + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + source_label=source_label, + target_label=target_label, + host=host, switch=switch, + extra_attributes=extra_attributes) + if self.inv.monitoring_setup_manager: + self.inv.monitoring_setup_manager.create_setup(link) diff --git a/app/discover/link_finders/find_links_for_instance_vnics.py b/app/discover/link_finders/find_links_for_instance_vnics.py new file mode 100644 index 0000000..7e0273d --- /dev/null +++ b/app/discover/link_finders/find_links_for_instance_vnics.py @@ -0,0 +1,60 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForInstanceVnics(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + self.log.info("adding links of type: instance-vnic") + vnics = self.inv.find_items({ + "environment": self.get_env(), + "type": "vnic", + "vnic_type": "instance_vnic" + }) + for v in vnics: + self.add_link_for_vnic(v) + + def add_link_for_vnic(self, v): + instance = self.inv.get_by_id(self.get_env(), v["instance_id"]) + if "network_info" not in instance: + self.log.warn("add_link_for_vnic: " + + "network_info missing in instance: %s ", + instance["id"]) + return + host = self.inv.get_by_id(self.get_env(), instance["host"]) + host_types = host["host_type"] + if "Network" not in host_types and "Compute" not in host_types: + return [] + source = instance["_id"] + source_id = instance["id"] + target = v["_id"] + target_id = v["id"] + link_type = "instance-vnic" + # find related network + network_name = None + network_id = None + for net in instance["network_info"]: + if net["devname"] == v["id"]: + network_name = net["network"]["label"] + network_id = net['network']['id'] + v['network'] = network_id + self.inv.set(v) + break + state = "up" # TBD + link_weight = 0 # TBD + attributes = {} if not network_id else {'network': network_id} + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, network_name, state, link_weight, + host=host["name"], + extra_attributes=attributes) diff --git a/app/discover/link_finders/find_links_for_oteps.py b/app/discover/link_finders/find_links_for_oteps.py new file mode 100644 index 0000000..b5d1667 --- /dev/null +++ b/app/discover/link_finders/find_links_for_oteps.py @@ -0,0 +1,88 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForOteps(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + self.log.info("adding link types: " + + "vedge-otep, otep-vconnector, otep-host_pnic") + oteps = self.inv.find_items({ + "environment": self.get_env(), + "type": "otep" + }) + for otep in oteps: + self.add_vedge_otep_link(otep) + self.add_otep_vconnector_link(otep) + self.add_otep_pnic_link(otep) + + def add_vedge_otep_link(self, otep): + vedge = self.inv.get_by_id(self.get_env(), otep["parent_id"]) + source = vedge["_id"] + source_id = vedge["id"] + target = otep["_id"] + target_id = otep["id"] + link_type = "vedge-otep" + link_name = vedge["name"] + "-otep" + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=vedge["host"]) + + def add_otep_vconnector_link(self, otep): + if "vconnector" not in otep: + return + vconnector = self.inv.find_items({ + "environment": self.get_env(), + "type": "vconnector", + "host": otep["host"], + "name": otep["vconnector"] + }, get_single=True) + if not vconnector: + return + source = otep["_id"] + source_id = otep["id"] + target = vconnector["_id"] + target_id = vconnector["id"] + link_type = "otep-vconnector" + link_name = otep["name"] + "-" + otep["vconnector"] + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=otep["host"]) + + def add_otep_pnic_link(self, otep): + pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "host_pnic", + "host": otep["host"], + "IP Address": otep["ip_address"] + }, get_single=True) + if not pnic: + return + source = otep["_id"] + source_id = otep["id"] + target = pnic["_id"] + target_id = pnic["id"] + link_type = "otep-host_pnic" + link_name = otep["host"] + "pnic" + pnic["name"] + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=otep["host"]) diff --git a/app/discover/link_finders/find_links_for_pnics.py b/app/discover/link_finders/find_links_for_pnics.py new file mode 100644 index 0000000..1f02426 --- /dev/null +++ b/app/discover/link_finders/find_links_for_pnics.py @@ -0,0 +1,132 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.link_finders.find_links import FindLinks +from utils.util import decode_aci_dn + + +class FindLinksForPnics(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + self.log.info("adding links of type: pnic-network, " + "host_pnic-switch_pnic, switch-host_pnic") + pnics = self.inv.find_items({ + "environment": self.get_env(), + "type": "host_pnic" + }) + for pnic in pnics: + self.add_pnic_network_links(pnic) + self.add_host_pnic_to_switch_pnic_link(pnic) + + self.log.info("adding links of type: switch_pnic-switch_pnic, " + "switch-switch_pnic") + pnics = self.inv.find_items({ + "environment": self.get_env(), + "type": "switch_pnic", + }) + for pnic in pnics: + self.add_switch_to_pnic_link(pnic) + if pnic["role"] == "uplink": + self.add_switch_pnic_to_switch_pnic_link(pnic) + + def add_pnic_network_links(self, pnic): + host = pnic["host"] + # find ports for that host, and fetch just the network ID + ports = self.inv.find_items({ + "environment": self.get_env(), + "type": "port", + "binding:host_id": host + }, {"network_id": 1, "id": 1}) + networks = {} + for port in ports: + networks[port["network_id"]] = 1 + for network_id in networks.keys(): + network = self.inv.get_by_id(self.get_env(), network_id) + if not network: + return + source = pnic["_id"] + source_id = pnic["id"] + target = network["_id"] + target_id = network["id"] + link_type = "host_pnic-network" + link_name = "Segment-" + str(network["provider:segmentation_id"]) \ + if "provider:segmentation_id" in network \ + else "Segment-None" + state = "up" if pnic["Link detected"] == "yes" else "down" + link_weight = 0 # TBD + attributes = {"network": target_id} + if "port_id" in pnic: + attributes['source_label'] = "port-" + pnic["port_id"] + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=host, + extra_attributes=attributes) + + def add_host_pnic_to_switch_pnic_link(self, host_pnic): + switch_pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "switch_pnic", + "mac_address": host_pnic["mac_address"]}, + get_single=True) + if not switch_pnic: + return + source = host_pnic["_id"] + source_id = host_pnic["id"] + target = switch_pnic["_id"] + target_id = switch_pnic["id"] + link_type = "host_pnic-switch_pnic" + link_name = "{}-{}".format(host_pnic['host'], + switch_pnic['parent_id']) + state = "up" if host_pnic["Link detected"] == "yes" else "down" + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=host_pnic['host']) + + def add_switch_pnic_to_switch_pnic_link(self, leaf_pnic): + spine_pnic = self.inv.get_by_id(self.get_env(), + leaf_pnic['connected_to']) + if not spine_pnic: + return + source = leaf_pnic["_id"] + source_id = leaf_pnic["id"] + target = spine_pnic["_id"] + target_id = spine_pnic["id"] + link_type = "switch_pnic-switch_pnic" + if_id_matches = re.search("(eth.*)$", source_id) + link_name = decode_aci_dn(if_id_matches.group(1)) + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + switch=leaf_pnic['switch']) + + def add_switch_to_pnic_link(self, pnic): + switch = self.inv.get_by_id(self.get_env(), pnic['parent_id']) + if not switch: + return + source = switch["_id"] + source_id = switch["id"] + target = pnic["_id"] + target_id = pnic["id"] + link_type = "switch-{}".format(pnic['type']) + link_name = "{}={}".format(switch["object_name"], pnic["object_name"]) + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + switch=switch['id']) diff --git a/app/discover/link_finders/find_links_for_vconnectors.py b/app/discover/link_finders/find_links_for_vconnectors.py new file mode 100644 index 0000000..edb351a --- /dev/null +++ b/app/discover/link_finders/find_links_for_vconnectors.py @@ -0,0 +1,91 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForVconnectors(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + vconnectors = self.inv.find_items({ + "environment": self.get_env(), + "type": "vconnector" + }) + self.log.info("adding links of type: vnic-vconnector, " + "vconnector-host_pnic") + for vconnector in vconnectors: + for interface in vconnector["interfaces_names"]: + self.add_vnic_vconnector_link(vconnector, interface) + self.add_vconnector_pnic_link(vconnector, interface) + + def add_vnic_vconnector_link(self, vconnector, interface_name): + mechanism_drivers = self.configuration.environment['mechanism_drivers'] + is_ovs = mechanism_drivers and mechanism_drivers[0] == 'OVS' + if is_ovs: + # interface ID for OVS + vnic = self.inv.get_by_id(self.get_env(), interface_name) + else: + # interface ID for VPP - match interface MAC address to vNIC MAC + interface = vconnector['interfaces'][interface_name] + if not interface or 'mac_address' not in interface: + return + vnic_mac = interface['mac_address'] + vnic = self.inv.get_by_field(self.get_env(), 'vnic', + 'mac_address', vnic_mac, + get_single=True) + if not vnic: + return + host = vnic["host"] + source = vnic["_id"] + source_id = vnic["id"] + target = vconnector["_id"] + target_id = vconnector["id"] + link_type = "vnic-vconnector" + link_name = vnic["mac_address"] + state = "up" # TBD + link_weight = 0 # TBD + attributes = {} + if 'network' in vnic: + attributes = {'network': vnic['network']} + vconnector['network'] = vnic['network'] + self.inv.set(vconnector) + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=host, + extra_attributes=attributes) + + def add_vconnector_pnic_link(self, vconnector, interface): + ifname = interface['name'] if isinstance(interface, dict) else interface + if "." in ifname: + ifname = ifname[:ifname.index(".")] + host = vconnector["host"] + pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "host_pnic", + "host": vconnector["host"], + "name": ifname + }, get_single=True) + if not pnic: + return + source = vconnector["_id"] + source_id = vconnector["id"] + target = pnic["_id"] + target_id = pnic["id"] + link_type = "vconnector-host_pnic" + link_name = pnic["name"] + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, + target, target_id, + link_type, link_name, state, link_weight, + host=host) diff --git a/app/discover/link_finders/find_links_for_vedges.py b/app/discover/link_finders/find_links_for_vedges.py new file mode 100644 index 0000000..f9719b4 --- /dev/null +++ b/app/discover/link_finders/find_links_for_vedges.py @@ -0,0 +1,128 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForVedges(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self): + self.log.info("adding link types: " + + "vnic-vedge, vconnector-vedge, vedge-host_pnic") + vedges = self.inv.find_items({ + "environment": self.get_env(), + "type": "vedge" + }) + for vedge in vedges: + ports = vedge["ports"] + for p in ports.values(): + self.add_link_for_vedge(vedge, p) + + def add_link_for_vedge(self, vedge, port): + vnic = self.inv.get_by_id(self.get_env(), + vedge['host'] + '-' + port["name"]) + if not vnic: + self.find_matching_vconnector(vedge, port) + self.find_matching_pnic(vedge, port) + return + source = vnic["_id"] + source_id = vnic["id"] + target = vedge["_id"] + target_id = vedge["id"] + link_type = "vnic-vedge" + link_name = vnic["name"] + "-" + vedge["name"] + if "tag" in port: + link_name += "-" + port["tag"] + state = "up" # TBD + link_weight = 0 # TBD + source_label = vnic["mac_address"] + target_label = port["id"] + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=vedge["host"], + extra_attributes={"source_label": source_label, + "target_label": target_label}) + + def find_matching_vconnector(self, vedge, port): + if self.configuration.has_network_plugin('VPP'): + vconnector_interface_name = port['name'] + else: + if not port["name"].startswith("qv"): + return + base_id = port["name"][3:] + vconnector_interface_name = "qvb" + base_id + vconnector = self.inv.find_items({ + "environment": self.get_env(), + "type": "vconnector", + "host": vedge['host'], + 'interfaces_names': vconnector_interface_name}, + get_single=True) + if not vconnector: + return + source = vconnector["_id"] + source_id = vconnector["id"] + target = vedge["_id"] + target_id = vedge["id"] + link_type = "vconnector-vedge" + link_name = "port-" + port["id"] + if "tag" in port: + link_name += "-" + port["tag"] + state = "up" # TBD + link_weight = 0 # TBD + source_label = vconnector_interface_name + target_label = port["name"] + mac_address = "Unknown" + attributes = {'mac_address': mac_address, 'source_label': source_label, + 'target_label': target_label} + for interface in vconnector['interfaces'].values(): + if vconnector_interface_name != interface['name']: + continue + if 'mac_address' not in interface: + continue + mac_address = interface['mac_address'] + attributes['mac_address'] = mac_address + break + if 'network' in vconnector: + attributes['network'] = vconnector['network'] + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=vedge["host"], + extra_attributes=attributes) + + def find_matching_pnic(self, vedge, port): + pname = port["name"] + if "pnic" in vedge: + if pname != vedge["pnic"]: + return + elif self.configuration.has_network_plugin('VPP'): + pass + pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "host_pnic", + "host": vedge["host"], + "name": pname + }, get_single=True) + if not pnic: + return + source = vedge["_id"] + source_id = vedge["id"] + target = pnic["_id"] + target_id = pnic["id"] + link_type = "vedge-host_pnic" + link_name = "Port-" + port["id"] + state = "up" if pnic["Link detected"] == "yes" else "down" + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, target, target_id, + link_type, link_name, state, link_weight, + host=vedge["host"]) diff --git a/app/discover/link_finders/find_links_for_vservice_vnics.py b/app/discover/link_finders/find_links_for_vservice_vnics.py new file mode 100644 index 0000000..ca9bc4a --- /dev/null +++ b/app/discover/link_finders/find_links_for_vservice_vnics.py @@ -0,0 +1,57 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindLinksForVserviceVnics(FindLinks): + def __init__(self): + super().__init__() + + def add_links(self, search=None): + self.log.info("adding links of type: vservice-vnic") + + if search is None: + search = {} + + search.update({"environment": self.get_env(), + "type": "vnic", + "vnic_type": "vservice_vnic"}) + + vnics = self.inv.find_items(search) + + for v in vnics: + self.add_link_for_vnic(v) + + def add_link_for_vnic(self, v): + host = self.inv.get_by_id(self.get_env(), v["host"]) + if "Network" not in host["host_type"]: + return + if "network" not in v: + return + network = self.inv.get_by_id(self.get_env(), v["network"]) + if network == []: + return + vservice_id = v["parent_id"] + vservice_id = vservice_id[:vservice_id.rindex('-')] + vservice = self.inv.get_by_id(self.get_env(), vservice_id) + source = vservice["_id"] + source_id = vservice_id + target = v["_id"] + target_id = v["id"] + link_type = "vservice-vnic" + link_name = network["name"] + state = "up" # TBD + link_weight = 0 # TBD + self.create_link(self.get_env(), + source, source_id, + target, target_id, + link_type, link_name, state, link_weight, + host=v["host"], + extra_attributes={'network': v['network']}) diff --git a/app/discover/link_finders/find_links_metadata_parser.py b/app/discover/link_finders/find_links_metadata_parser.py new file mode 100644 index 0000000..1f28262 --- /dev/null +++ b/app/discover/link_finders/find_links_metadata_parser.py @@ -0,0 +1,57 @@ +from utils.metadata_parser import MetadataParser +from utils.util import ClassResolver + + +class FindLinksMetadataParser(MetadataParser): + + FINDERS_FILE = "link_finders.json" + + FINDERS_PACKAGE = "finders_package" + BASE_FINDER = "base_finder" + LINK_FINDERS = "link_finders" + + def __init__(self): + super().__init__() + self.finders_package = None + self.base_finder = None + self.link_finders = [] + + def validate_link_finder(self, finder_class): + try: + module_name = ClassResolver.get_module_file_by_class_name(finder_class) + instance = ClassResolver\ + .get_instance_of_class(package_name=self.finders_package, + module_name=module_name, + class_name=finder_class) + except ValueError: + instance = None + + if instance: + self.link_finders.append(instance) + else: + self.add_error('Failed to import link finder class "{}"' + .format(finder_class)) + + def validate_metadata(self, metadata: dict): + super().validate_metadata(metadata) + self.finders_package = metadata[self.FINDERS_PACKAGE] + self.base_finder = metadata[self.BASE_FINDER] + base_finder_module = ClassResolver\ + .get_module_file_by_class_name(self.base_finder) + + base_finder_class = ClassResolver.get_class_name_by_module( + ".".join((self.finders_package, base_finder_module))) + + if not base_finder_class: + self.add_error("Couldn't find base link finder class") + return + + for link_finder in metadata[self.LINK_FINDERS]: + self.validate_link_finder(finder_class=link_finder) + metadata[self.LINK_FINDERS] = self.link_finders + + return len(self.errors) == 0 + + def get_required_fields(self) -> list: + return [self.FINDERS_PACKAGE, self.BASE_FINDER, self.LINK_FINDERS] + -- cgit 1.2.3-korg