From 9d51c29acfb81ae16c72a1675d7b021d032dbdfb Mon Sep 17 00:00:00 2001 From: yayogev Date: Fri, 11 Aug 2017 00:35:14 +0300 Subject: US2853 rename pnic to host_pnic & switch_pnic fixed clique finder to work with switch-switch link type Change-Id: I4f815f77e6b4ea36669dfb58bb5f5e914fe3e1ac Signed-off-by: yayogev --- app/config/scanners.json | 8 +- app/discover/clique_finder.py | 22 +- .../fetchers/aci/aci_fetch_leaf_to_spine_pnics.py | 304 +++-- app/discover/fetchers/aci/aci_fetch_switch_pnic.py | 215 ++- app/discover/fetchers/cli/cli_fetch_host_pnics.py | 3 +- .../fetchers/cli/cli_fetch_host_pnics_vpp.py | 2 +- app/discover/fetchers/db/db_fetch_vedges_ovs.py | 2 +- app/discover/find_links_for_oteps.py | 6 +- app/discover/find_links_for_pnics.py | 27 +- app/discover/find_links_for_vconnectors.py | 7 +- app/discover/find_links_for_vedges.py | 6 +- app/install/db/attributes_for_hover_on_data.json | 178 +-- app/install/db/clique_types.json | 112 +- app/install/db/constants.json | 1368 ++++++++++---------- app/install/db/link_types.json | 233 ++-- app/monitoring/setup/monitoring_pnic.py | 3 +- app/monitoring/setup/monitoring_setup_manager.py | 2 +- app/test/api/responders_test/test_data/base.py | 14 +- .../api/responders_test/test_data/clique_types.py | 4 +- app/test/api/responders_test/test_data/cliques.py | 6 +- app/test/api/responders_test/test_data/links.py | 4 +- app/test/scan/test_data/scanner.py | 10 +- 22 files changed, 1257 insertions(+), 1279 deletions(-) diff --git a/app/config/scanners.json b/app/config/scanners.json index feaafcb..fa65891 100644 --- a/app/config/scanners.json +++ b/app/config/scanners.json @@ -211,7 +211,7 @@ ], "ScanPnicsRoot": [ { - "type": "pnic", + "type": "host_pnic", "environment_condition": { "mechanism_drivers": [ "OVS", @@ -224,14 +224,14 @@ ], "ScanHostPnic": [ { - "type": "pnic", + "type": "switch_pnic", "fetcher": "AciFetchSwitchPnic", "children_scanner": "ScanSpines" } ], "ScanSpines": [ { - "type": "pnic", + "type": "switch_pnic", "fetcher": "AciFetchLeafToSpinePnics" } ], @@ -359,7 +359,7 @@ ], "ScanVppPnicsRoot": [ { - "type": "pnic", + "type": "host_pnic", "fetcher": "CliFetchHostPnicsVpp", "environment_condition": { "mechanism_drivers": "VPP" diff --git a/app/discover/clique_finder.py b/app/discover/clique_finder.py index 9b5aad2..4c4e636 100644 --- a/app/discover/clique_finder.py +++ b/app/discover/clique_finder.py @@ -100,11 +100,15 @@ class CliqueFinder(Fetcher): link_type_parts = link_type.split('-') link_type_parts.reverse() link_type_reversed = '-'.join(link_type_parts) - matches = self.links.find_one({ - "environment": self.env, - "link_type": link_type_reversed - }) - reversed = True if matches else False + self_linked = link_type == link_type_reversed + if self_linked: + reversed = False + else: + matches = self.links.find_one({ + "environment": self.env, + "link_type": link_type_reversed + }) + reversed = True if matches else False if reversed: link_type = link_type_reversed from_type = link_type[:link_type.index("-")] @@ -115,6 +119,7 @@ class CliqueFinder(Fetcher): if match_type not in nodes_of_type.keys(): continue other_side_type = to_type if not reversed else from_type + nodes_to_add = {} for match_point in nodes_of_type[match_type].keys(): matches = self.links.find({ "environment": self.env, @@ -130,9 +135,10 @@ class CliqueFinder(Fetcher): clique["links"].append(id) clique["links_detailed"].append(link) other_side_point = str(link[other_side]) - if other_side_type not in nodes_of_type: - nodes_of_type[other_side_type] = {} - nodes_of_type[other_side_type][other_side_point] = 1 + nodes_to_add[other_side_point] = 1 + if other_side_type not in nodes_of_type: + nodes_of_type[other_side_type] = {} + nodes_of_type[other_side_type].update(nodes_to_add) # after adding the links to the clique, create/update the clique if not clique["links"]: diff --git a/app/discover/fetchers/aci/aci_fetch_leaf_to_spine_pnics.py b/app/discover/fetchers/aci/aci_fetch_leaf_to_spine_pnics.py index 463d4d8..992df77 100644 --- a/app/discover/fetchers/aci/aci_fetch_leaf_to_spine_pnics.py +++ b/app/discover/fetchers/aci/aci_fetch_leaf_to_spine_pnics.py @@ -1,153 +1,151 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -import re - -from discover.fetchers.aci.aci_access import AciAccess, aci_config_required -from utils.inventory_mgr import InventoryMgr -from utils.util import decode_aci_dn, encode_aci_dn, get_object_path_part - - -# Fetches and adds to database: -# 1. ACI Switches with role "spine" -# -# Returns: -# 1. ACI Switch uplink pnics that belong to the "leaf" switch -# 2. ACI Switch downlink pnics that belong to "spine" switches mentioned above -# and are connected to the "leaf" switch -class AciFetchLeafToSpinePnics(AciAccess): - - def __init__(self): - super().__init__() - self.inv = InventoryMgr() - - def fetch_switches_by_role(self, role_name): - query_filter = {"query-target-filter": - "eq(fabricNode.role, \"{}\")".format(role_name)} - switches = self.fetch_objects_by_class("fabricNode", query_filter) - return [switch["attributes"] for switch in switches] - - def fetch_adjacent_connections(self, device_id): - dn = "/".join((device_id, "sys")) - - response = self.fetch_mo_data(dn, - {"query-target": "subtree", - "target-subtree-class": "lldpAdjEp"}) - - connections = self.get_objects_by_field_names(response, - "lldpAdjEp", "attributes") - return connections - - # Returns: - # List of: - # 1. Switches with role "spine" - # 2. Downlink pnic id for spine switch - # 3. Uplink pnic id for leaf switch - def fetch_spines_and_pnics_by_leaf_id(self, leaf_id): - spine_switches = self.fetch_switches_by_role("spine") - adjacent_devices = self.fetch_adjacent_connections(leaf_id) - spines = [] - for spine in spine_switches: - # Check if spine switch is connected to current leaf switch - connection = next((d for d in adjacent_devices - if spine["name"] == d["sysName"]), - None) - if connection: - try: - # Extract pnics from adjacency data - uplink_pnic = re.match(".*\[(.+?)\].*", - connection["dn"]).group(1) - downlink_pnic = re.match(".*\[(.+?)\].*", - connection["portDesc"]).group(1) - spines.append({ - "device": spine, - "downlink_pnic": downlink_pnic, - "uplink_pnic": uplink_pnic - }) - except AttributeError: - continue # TODO: probably raise an exception - - return spines - - @aci_config_required(default=[]) - def get(self, db_leaf_pnic_id): - environment = self.get_env() - leaf_pnic = self.inv.get_by_id(environment=environment, - item_id=db_leaf_pnic_id) - leaf_switch_id = leaf_pnic['switch'] - - # Decode aci leaf switch id from db format - aci_leaf_pnic_id = decode_aci_dn(db_leaf_pnic_id) - aci_leaf_id = re.match("switch-(.+?)-leaf", aci_leaf_pnic_id).group(1) - - # Fetch all leaf-to-spine connectivity data - spines_with_pnics = self.fetch_spines_and_pnics_by_leaf_id(aci_leaf_id) - pnics = [] - for spine_with_pnic in spines_with_pnics: - spine = spine_with_pnic["device"] - downlink_pnic_id = spine_with_pnic["downlink_pnic"] - uplink_pnic_id = spine_with_pnic["uplink_pnic"] - - # Add spine switch to db if it's not there yet - spine_id_match = re.match("topology/(.+)", spine["dn"]) - if not spine_id_match: - raise ValueError("Failed to fetch spine switch id " - "from switch dn: {}".format(spine["dn"])) - - aci_spine_id = spine_id_match.group(1) - db_spine_id = "-".join(("switch", encode_aci_dn(aci_spine_id), - spine["role"])) - if not self.inv.get_by_id(environment, db_spine_id): - spine_json = { - "id": db_spine_id, - "type": "switch", - "aci_document": spine - } - # Region name is the same as region id - region_id = get_object_path_part(leaf_pnic["name_path"], - "Regions") - region = self.inv.get_by_id(environment, region_id) - self.inv.save_inventory_object(o=spine_json, parent=region, - environment=environment) - - # Add downlink and uplink pnics to results list, - # including their mutual connection data - # (see "connected_to" field). - db_downlink_pnic_id = "-".join((db_spine_id, - encode_aci_dn(downlink_pnic_id))) - db_uplink_pnic_id = "-".join((leaf_pnic["switch"], - encode_aci_dn(uplink_pnic_id))) - - downlink_pnic_json = { - "id": db_downlink_pnic_id, - "type": "pnic", - "role": "downlink", - "pnic_type": "switch", - "connected_to": db_uplink_pnic_id, - "switch": db_spine_id, - "parent_id": db_spine_id, - "parent_type": "switch", - "aci_document": {} # TODO: what can we add here? - } - - uplink_pnic_json = { - "id": db_uplink_pnic_id, - "type": "pnic", - "role": "uplink", - "pnic_type": "switch", - "connected_to": db_downlink_pnic_id, - "switch": leaf_switch_id, - "parent_id": leaf_switch_id, - "parent_type": "switch", - "aci_document": {} # TODO: what can we add here? - } - - pnics.extend([downlink_pnic_json, uplink_pnic_json]) - - return pnics +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.aci.aci_access import AciAccess, aci_config_required +from utils.inventory_mgr import InventoryMgr +from utils.util import decode_aci_dn, encode_aci_dn, get_object_path_part + + +# Fetches and adds to database: +# 1. ACI Switches with role "spine" +# +# Returns: +# 1. ACI Switch uplink pnics that belong to the "leaf" switch +# 2. ACI Switch downlink pnics that belong to "spine" switches mentioned above +# and are connected to the "leaf" switch +class AciFetchLeafToSpinePnics(AciAccess): + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def fetch_switches_by_role(self, role_name): + query_filter = {"query-target-filter": + "eq(fabricNode.role, \"{}\")".format(role_name)} + switches = self.fetch_objects_by_class("fabricNode", query_filter) + return [switch["attributes"] for switch in switches] + + def fetch_adjacent_connections(self, device_id): + dn = "/".join((device_id, "sys")) + + response = self.fetch_mo_data(dn, + {"query-target": "subtree", + "target-subtree-class": "lldpAdjEp"}) + + connections = self.get_objects_by_field_names(response, + "lldpAdjEp", "attributes") + return connections + + # Returns: + # List of: + # 1. Switches with role "spine" + # 2. Downlink pnic id for spine switch + # 3. Uplink pnic id for leaf switch + def fetch_spines_and_pnics_by_leaf_id(self, leaf_id): + spine_switches = self.fetch_switches_by_role("spine") + adjacent_devices = self.fetch_adjacent_connections(leaf_id) + spines = [] + for spine in spine_switches: + # Check if spine switch is connected to current leaf switch + connection = next((d for d in adjacent_devices + if spine["name"] == d["sysName"]), + None) + if connection: + try: + # Extract pnics from adjacency data + uplink_pnic = re.match(".*\[(.+?)\].*", + connection["dn"]).group(1) + downlink_pnic = re.match(".*\[(.+?)\].*", + connection["portDesc"]).group(1) + spines.append({ + "device": spine, + "downlink_pnic": downlink_pnic, + "uplink_pnic": uplink_pnic + }) + except AttributeError: + continue # TODO: probably raise an exception + + return spines + + @aci_config_required(default=[]) + def get(self, db_leaf_pnic_id): + environment = self.get_env() + leaf_pnic = self.inv.get_by_id(environment=environment, + item_id=db_leaf_pnic_id) + leaf_switch_id = leaf_pnic['switch'] + + # Decode aci leaf switch id from db format + aci_leaf_pnic_id = decode_aci_dn(db_leaf_pnic_id) + aci_leaf_id = re.match("switch-(.+?)-leaf", aci_leaf_pnic_id).group(1) + + # Fetch all leaf-to-spine connectivity data + spines_with_pnics = self.fetch_spines_and_pnics_by_leaf_id(aci_leaf_id) + pnics = [] + for spine_with_pnic in spines_with_pnics: + spine = spine_with_pnic["device"] + downlink_pnic_id = spine_with_pnic["downlink_pnic"] + uplink_pnic_id = spine_with_pnic["uplink_pnic"] + + # Add spine switch to db if it's not there yet + spine_id_match = re.match("topology/(.+)", spine["dn"]) + if not spine_id_match: + raise ValueError("Failed to fetch spine switch id " + "from switch dn: {}".format(spine["dn"])) + + aci_spine_id = spine_id_match.group(1) + db_spine_id = "-".join(("switch", encode_aci_dn(aci_spine_id), + spine["role"])) + if not self.inv.get_by_id(environment, db_spine_id): + spine_json = { + "id": db_spine_id, + "type": "switch", + "aci_document": spine + } + # Region name is the same as region id + region_id = get_object_path_part(leaf_pnic["name_path"], + "Regions") + region = self.inv.get_by_id(environment, region_id) + self.inv.save_inventory_object(o=spine_json, parent=region, + environment=environment) + + # Add downlink and uplink pnics to results list, + # including their mutual connection data + # (see "connected_to" field). + db_downlink_pnic_id = "-".join((db_spine_id, + encode_aci_dn(downlink_pnic_id))) + db_uplink_pnic_id = "-".join((leaf_pnic["switch"], + encode_aci_dn(uplink_pnic_id))) + + downlink_pnic_json = { + "id": db_downlink_pnic_id, + "type": "switch_pnic", + "role": "downlink", + "connected_to": db_uplink_pnic_id, + "switch": db_spine_id, + "parent_id": db_spine_id, + "parent_type": "switch", + "aci_document": {} # TODO: what can we add here? + } + + uplink_pnic_json = { + "id": db_uplink_pnic_id, + "type": "switch_pnic", + "role": "uplink", + "connected_to": db_downlink_pnic_id, + "switch": leaf_switch_id, + "parent_id": leaf_switch_id, + "parent_type": "switch", + "aci_document": {} # TODO: what can we add here? + } + + pnics.extend([downlink_pnic_json, uplink_pnic_json]) + + return pnics diff --git a/app/discover/fetchers/aci/aci_fetch_switch_pnic.py b/app/discover/fetchers/aci/aci_fetch_switch_pnic.py index 09157b6..cd82d7e 100644 --- a/app/discover/fetchers/aci/aci_fetch_switch_pnic.py +++ b/app/discover/fetchers/aci/aci_fetch_switch_pnic.py @@ -1,108 +1,107 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -import re - -from discover.fetchers.aci.aci_access import AciAccess, aci_config_required -from utils.inventory_mgr import InventoryMgr -from utils.util import encode_aci_dn, get_object_path_part - - -# Fetches and adds to database: -# 1. ACI Switch -# -# Returns: -# 1. ACI Switch pnic that belongs to the ACI Switch (mentioned above) -# and is connected to Calipso host pnic. -class AciFetchSwitchPnic(AciAccess): - - def __init__(self): - super().__init__() - self.inv = InventoryMgr() - - def fetch_pnics_by_mac_address(self, mac_address): - mac_filter = "eq(epmMacEp.addr,\"{}\")".format(mac_address) - # We are only interested in Ethernet interfaces - pnic_filter = "wcard(epmMacEp.ifId, \"eth\")" - query_filter = {"query-target-filter": - "and({},{})".format(mac_filter, pnic_filter)} - - pnics = self.fetch_objects_by_class("epmMacEp", query_filter) - - return [pnic["attributes"] for pnic in pnics] - - def fetch_switch_by_id(self, switch_id): - dn = "/".join((switch_id, "sys")) - response = self.fetch_mo_data(dn) - # Unwrap switches - switch_data = self.get_objects_by_field_names(response, "topSystem", - "attributes") - return switch_data[0] if switch_data else None - - @aci_config_required(default=[]) - def get(self, pnic_id): - environment = self.get_env() - pnic = self.inv.get_by_id(environment=environment, item_id=pnic_id) - if not pnic: - return [] - mac_address = pnic.get("mac_address") - if not mac_address: - return [] - - # Query ACI for related switch pnic - leaf_pnics = self.fetch_pnics_by_mac_address(mac_address) - if not leaf_pnics: - return [] - leaf_pnic = leaf_pnics[0] - - # Prepare and save switch data in inventory - leaf_id_match = re.match("topology/(.+)/sys", leaf_pnic["dn"]) - if not leaf_id_match: - raise ValueError("Failed to fetch leaf switch id from pnic dn: {}" - .format(leaf_pnic["dn"])) - - aci_leaf_id = leaf_id_match.group(1) - leaf_data = self.fetch_switch_by_id(aci_leaf_id) - if not leaf_data: - self.log.warning("No switch found for switch pnic dn: {}" - .format(leaf_pnic["dn"])) - return [] - - db_leaf_id = "-".join(("switch", encode_aci_dn(aci_leaf_id), - leaf_data["role"])) - if not self.inv.get_by_id(environment, db_leaf_id): - leaf_json = { - "id": db_leaf_id, - "ip_address": leaf_data["address"], - "type": "switch", - "host": db_leaf_id, - "aci_document": leaf_data - } - # Region name is the same as region id - region_id = get_object_path_part(pnic["name_path"], "Regions") - region = self.inv.get_by_id(environment, region_id) - self.inv.save_inventory_object(o=leaf_json, parent=region, - environment=environment) - - # Prepare pnic json for results list - db_pnic_id = "-".join((db_leaf_id, - encode_aci_dn(leaf_pnic["ifId"]), - mac_address)) - pnic_json = { - "id": db_pnic_id, - "type": "pnic", - "role": "hostlink", - "parent_id": db_leaf_id, - "parent_type": "switch", - "pnic_type": "switch", - "mac_address": mac_address, - "switch": db_leaf_id, - "aci_document": leaf_pnic - } - return [pnic_json] +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.aci.aci_access import AciAccess, aci_config_required +from utils.inventory_mgr import InventoryMgr +from utils.util import encode_aci_dn, get_object_path_part + + +# Fetches and adds to database: +# 1. ACI Switch +# +# Returns: +# 1. ACI Switch pnic that belongs to the ACI Switch (mentioned above) +# and is connected to Calipso host pnic. +class AciFetchSwitchPnic(AciAccess): + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def fetch_pnics_by_mac_address(self, mac_address): + mac_filter = "eq(epmMacEp.addr,\"{}\")".format(mac_address) + # We are only interested in Ethernet interfaces + pnic_filter = "wcard(epmMacEp.ifId, \"eth\")" + query_filter = {"query-target-filter": + "and({},{})".format(mac_filter, pnic_filter)} + + pnics = self.fetch_objects_by_class("epmMacEp", query_filter) + + return [pnic["attributes"] for pnic in pnics] + + def fetch_switch_by_id(self, switch_id): + dn = "/".join((switch_id, "sys")) + response = self.fetch_mo_data(dn) + # Unwrap switches + switch_data = self.get_objects_by_field_names(response, "topSystem", + "attributes") + return switch_data[0] if switch_data else None + + @aci_config_required(default=[]) + def get(self, pnic_id): + environment = self.get_env() + pnic = self.inv.get_by_id(environment=environment, item_id=pnic_id) + if not pnic: + return [] + mac_address = pnic.get("mac_address") + if not mac_address: + return [] + + # Query ACI for related switch pnic + leaf_pnics = self.fetch_pnics_by_mac_address(mac_address) + if not leaf_pnics: + return [] + leaf_pnic = leaf_pnics[0] + + # Prepare and save switch data in inventory + leaf_id_match = re.match("topology/(.+)/sys", leaf_pnic["dn"]) + if not leaf_id_match: + raise ValueError("Failed to fetch leaf switch id from pnic dn: {}" + .format(leaf_pnic["dn"])) + + aci_leaf_id = leaf_id_match.group(1) + leaf_data = self.fetch_switch_by_id(aci_leaf_id) + if not leaf_data: + self.log.warning("No switch found for switch pnic dn: {}" + .format(leaf_pnic["dn"])) + return [] + + db_leaf_id = "-".join(("switch", encode_aci_dn(aci_leaf_id), + leaf_data["role"])) + if not self.inv.get_by_id(environment, db_leaf_id): + leaf_json = { + "id": db_leaf_id, + "ip_address": leaf_data["address"], + "type": "switch", + "host": db_leaf_id, + "aci_document": leaf_data + } + # Region name is the same as region id + region_id = get_object_path_part(pnic["name_path"], "Regions") + region = self.inv.get_by_id(environment, region_id) + self.inv.save_inventory_object(o=leaf_json, parent=region, + environment=environment) + + # Prepare pnic json for results list + db_pnic_id = "-".join((db_leaf_id, + encode_aci_dn(leaf_pnic["ifId"]), + mac_address)) + pnic_json = { + "id": db_pnic_id, + "type": "switch_pnic", + "role": "hostlink", + "parent_id": db_leaf_id, + "parent_type": "switch", + "mac_address": mac_address, + "switch": db_leaf_id, + "aci_document": leaf_pnic + } + return [pnic_json] diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py index 86aa693..3516e25 100644 --- a/app/discover/fetchers/cli/cli_fetch_host_pnics.py +++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py @@ -71,8 +71,7 @@ class CliFetchHostPnics(CliAccess): "host": host_id, "name": id, "local_name": interface_name, - "lines": [], - "pnic_type": "host" + "lines": [] } self.handle_line(interface, line_remainder) if '