Merge "add parent_type to switch pnics, =switch"

2 files changed, 261 insertions, 258 deletions

diff --git a/app/discover/fetchers/aci/aci_fetch_leaf_to_spine_pnics.py b/app/discover/fetchers/aci/aci_fetch_leaf_to_spine_pnics.py
index c556459..463d4d8 100644
--- a/app/discover/fetchers/aci/aci_fetch_leaf_to_spine_pnics.py
+++ b/app/discover/fetchers/aci/aci_fetch_leaf_to_spine_pnics.py
@@ -1,151 +1,153 @@
-###############################################################################
-# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
-# and others                                                                  #
-#                                                                             #
-# All rights reserved. This program and the accompanying materials            #
-# are made available under the terms of the Apache License, Version 2.0       #
-# which accompanies this distribution, and is available at                    #
-# http://www.apache.org/licenses/LICENSE-2.0                                  #
-###############################################################################
-import re
-
-from discover.fetchers.aci.aci_access import AciAccess, aci_config_required
-from utils.inventory_mgr import InventoryMgr
-from utils.util import decode_aci_dn, encode_aci_dn, get_object_path_part
-
-
-# Fetches and adds to database:
-# 1. ACI Switches with role "spine"
-#
-# Returns:
-# 1. ACI Switch uplink pnics that belong to the "leaf" switch
-# 2. ACI Switch downlink pnics that belong to "spine" switches mentioned above
-# and are connected to the "leaf" switch
-class AciFetchLeafToSpinePnics(AciAccess):
-
-    def __init__(self):
-        super().__init__()
-        self.inv = InventoryMgr()
-
-    def fetch_switches_by_role(self, role_name):
-        query_filter = {"query-target-filter":
-                        "eq(fabricNode.role, \"{}\")".format(role_name)}
-        switches = self.fetch_objects_by_class("fabricNode", query_filter)
-        return [switch["attributes"] for switch in switches]
-
-    def fetch_adjacent_connections(self, device_id):
-        dn = "/".join((device_id, "sys"))
-
-        response = self.fetch_mo_data(dn,
-                                      {"query-target": "subtree",
-                                       "target-subtree-class": "lldpAdjEp"})
-
-        connections = self.get_objects_by_field_names(response,
-                                                      "lldpAdjEp", "attributes")
-        return connections
-
-    # Returns:
-    # List of:
-    # 1. Switches with role "spine"
-    # 2. Downlink pnic id for spine switch
-    # 3. Uplink pnic id for leaf switch
-    def fetch_spines_and_pnics_by_leaf_id(self, leaf_id):
-        spine_switches = self.fetch_switches_by_role("spine")
-        adjacent_devices = self.fetch_adjacent_connections(leaf_id)
-        spines = []
-        for spine in spine_switches:
-            # Check if spine switch is connected to current leaf switch
-            connection = next((d for d in adjacent_devices
-                               if spine["name"] == d["sysName"]),
-                              None)
-            if connection:
-                try:
-                    # Extract pnics from adjacency data
-                    uplink_pnic = re.match(".*\[(.+?)\].*",
-                                           connection["dn"]).group(1)
-                    downlink_pnic = re.match(".*\[(.+?)\].*",
-                                             connection["portDesc"]).group(1)
-                    spines.append({
-                        "device": spine,
-                        "downlink_pnic": downlink_pnic,
-                        "uplink_pnic": uplink_pnic
-                    })
-                except AttributeError:
-                    continue  # TODO: probably raise an exception
-
-        return spines
-
-    @aci_config_required(default=[])
-    def get(self, db_leaf_pnic_id):
-        environment = self.get_env()
-        leaf_pnic = self.inv.get_by_id(environment=environment,
-                                       item_id=db_leaf_pnic_id)
-        leaf_switch_id = leaf_pnic['switch']
-
-        # Decode aci leaf switch id from db format
-        aci_leaf_pnic_id = decode_aci_dn(db_leaf_pnic_id)
-        aci_leaf_id = re.match("switch-(.+?)-leaf", aci_leaf_pnic_id).group(1)
-
-        # Fetch all leaf-to-spine connectivity data
-        spines_with_pnics = self.fetch_spines_and_pnics_by_leaf_id(aci_leaf_id)
-        pnics = []
-        for spine_with_pnic in spines_with_pnics:
-            spine = spine_with_pnic["device"]
-            downlink_pnic_id = spine_with_pnic["downlink_pnic"]
-            uplink_pnic_id = spine_with_pnic["uplink_pnic"]
-
-            # Add spine switch to db if it's not there yet
-            spine_id_match = re.match("topology/(.+)", spine["dn"])
-            if not spine_id_match:
-                raise ValueError("Failed to fetch spine switch id "
-                                 "from switch dn: {}".format(spine["dn"]))
-
-            aci_spine_id = spine_id_match.group(1)
-            db_spine_id = "-".join(("switch", encode_aci_dn(aci_spine_id),
-                                    spine["role"]))
-            if not self.inv.get_by_id(environment, db_spine_id):
-                spine_json = {
-                    "id": db_spine_id,
-                    "type": "switch",
-                    "aci_document": spine
-                }
-                # Region name is the same as region id
-                region_id = get_object_path_part(leaf_pnic["name_path"],
-                                                 "Regions")
-                region = self.inv.get_by_id(environment, region_id)
-                self.inv.save_inventory_object(o=spine_json, parent=region,
-                                               environment=environment)
-
-            # Add downlink and uplink pnics to results list,
-            # including their mutual connection data
-            # (see "connected_to" field).
-            db_downlink_pnic_id = "-".join((db_spine_id,
-                                            encode_aci_dn(downlink_pnic_id)))
-            db_uplink_pnic_id = "-".join((leaf_pnic["switch"],
-                                          encode_aci_dn(uplink_pnic_id)))
-
-            downlink_pnic_json = {
-                "id": db_downlink_pnic_id,
-                "type": "pnic",
-                "role": "downlink",
-                "pnic_type": "switch",
-                "connected_to": db_uplink_pnic_id,
-                "switch": db_spine_id,
-                "parent_id": db_spine_id,
-                "aci_document": {}  # TODO: what can we add here?
-            }
-
-            uplink_pnic_json = {
-                "id": db_uplink_pnic_id,
-                "type": "pnic",
-                "role": "uplink",
-                "pnic_type": "switch",
-                "connected_to": db_downlink_pnic_id,
-                "switch": leaf_switch_id,
-                "parent_id": leaf_switch_id,
-                "aci_document": {}  # TODO: what can we add here?
-            }
-
-            pnics.extend([downlink_pnic_json, uplink_pnic_json])
-
-        return pnics
+###############################################################################

+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #

+# and others                                                                  #

+#                                                                             #

+# All rights reserved. This program and the accompanying materials            #

+# are made available under the terms of the Apache License, Version 2.0       #

+# which accompanies this distribution, and is available at                    #

+# http://www.apache.org/licenses/LICENSE-2.0                                  #

+###############################################################################

+import re

+

+from discover.fetchers.aci.aci_access import AciAccess, aci_config_required

+from utils.inventory_mgr import InventoryMgr

+from utils.util import decode_aci_dn, encode_aci_dn, get_object_path_part

+

+

+# Fetches and adds to database:

+# 1. ACI Switches with role "spine"

+#

+# Returns:

+# 1. ACI Switch uplink pnics that belong to the "leaf" switch

+# 2. ACI Switch downlink pnics that belong to "spine" switches mentioned above

+# and are connected to the "leaf" switch

+class AciFetchLeafToSpinePnics(AciAccess):

+

+    def __init__(self):

+        super().__init__()

+        self.inv = InventoryMgr()

+

+    def fetch_switches_by_role(self, role_name):

+        query_filter = {"query-target-filter":

+                        "eq(fabricNode.role, \"{}\")".format(role_name)}

+        switches = self.fetch_objects_by_class("fabricNode", query_filter)

+        return [switch["attributes"] for switch in switches]

+

+    def fetch_adjacent_connections(self, device_id):

+        dn = "/".join((device_id, "sys"))

+

+        response = self.fetch_mo_data(dn,

+                                      {"query-target": "subtree",

+                                       "target-subtree-class": "lldpAdjEp"})

+

+        connections = self.get_objects_by_field_names(response,

+                                                      "lldpAdjEp", "attributes")

+        return connections

+

+    # Returns:

+    # List of:

+    # 1. Switches with role "spine"

+    # 2. Downlink pnic id for spine switch

+    # 3. Uplink pnic id for leaf switch

+    def fetch_spines_and_pnics_by_leaf_id(self, leaf_id):

+        spine_switches = self.fetch_switches_by_role("spine")

+        adjacent_devices = self.fetch_adjacent_connections(leaf_id)

+        spines = []

+        for spine in spine_switches:

+            # Check if spine switch is connected to current leaf switch

+            connection = next((d for d in adjacent_devices

+                               if spine["name"] == d["sysName"]),

+                              None)

+            if connection:

+                try:

+                    # Extract pnics from adjacency data

+                    uplink_pnic = re.match(".*\[(.+?)\].*",

+                                           connection["dn"]).group(1)

+                    downlink_pnic = re.match(".*\[(.+?)\].*",

+                                             connection["portDesc"]).group(1)

+                    spines.append({

+                        "device": spine,

+                        "downlink_pnic": downlink_pnic,

+                        "uplink_pnic": uplink_pnic

+                    })

+                except AttributeError:

+                    continue  # TODO: probably raise an exception

+

+        return spines

+

+    @aci_config_required(default=[])

+    def get(self, db_leaf_pnic_id):

+        environment = self.get_env()

+        leaf_pnic = self.inv.get_by_id(environment=environment,

+                                       item_id=db_leaf_pnic_id)

+        leaf_switch_id = leaf_pnic['switch']

+

+        # Decode aci leaf switch id from db format

+        aci_leaf_pnic_id = decode_aci_dn(db_leaf_pnic_id)

+        aci_leaf_id = re.match("switch-(.+?)-leaf", aci_leaf_pnic_id).group(1)

+

+        # Fetch all leaf-to-spine connectivity data

+        spines_with_pnics = self.fetch_spines_and_pnics_by_leaf_id(aci_leaf_id)

+        pnics = []

+        for spine_with_pnic in spines_with_pnics:

+            spine = spine_with_pnic["device"]

+            downlink_pnic_id = spine_with_pnic["downlink_pnic"]

+            uplink_pnic_id = spine_with_pnic["uplink_pnic"]

+

+            # Add spine switch to db if it's not there yet

+            spine_id_match = re.match("topology/(.+)", spine["dn"])

+            if not spine_id_match:

+                raise ValueError("Failed to fetch spine switch id "

+                                 "from switch dn: {}".format(spine["dn"]))

+

+            aci_spine_id = spine_id_match.group(1)

+            db_spine_id = "-".join(("switch", encode_aci_dn(aci_spine_id),

+                                    spine["role"]))

+            if not self.inv.get_by_id(environment, db_spine_id):

+                spine_json = {

+                    "id": db_spine_id,

+                    "type": "switch",

+                    "aci_document": spine

+                }

+                # Region name is the same as region id

+                region_id = get_object_path_part(leaf_pnic["name_path"],

+                                                 "Regions")

+                region = self.inv.get_by_id(environment, region_id)

+                self.inv.save_inventory_object(o=spine_json, parent=region,

+                                               environment=environment)

+

+            # Add downlink and uplink pnics to results list,

+            # including their mutual connection data

+            # (see "connected_to" field).

+            db_downlink_pnic_id = "-".join((db_spine_id,

+                                            encode_aci_dn(downlink_pnic_id)))

+            db_uplink_pnic_id = "-".join((leaf_pnic["switch"],

+                                          encode_aci_dn(uplink_pnic_id)))

+

+            downlink_pnic_json = {

+                "id": db_downlink_pnic_id,

+                "type": "pnic",

+                "role": "downlink",

+                "pnic_type": "switch",

+                "connected_to": db_uplink_pnic_id,

+                "switch": db_spine_id,

+                "parent_id": db_spine_id,

+                "parent_type": "switch",

+                "aci_document": {}  # TODO: what can we add here?

+            }

+

+            uplink_pnic_json = {

+                "id": db_uplink_pnic_id,

+                "type": "pnic",

+                "role": "uplink",

+                "pnic_type": "switch",

+                "connected_to": db_downlink_pnic_id,

+                "switch": leaf_switch_id,

+                "parent_id": leaf_switch_id,

+                "parent_type": "switch",

+                "aci_document": {}  # TODO: what can we add here?

+            }

+

+            pnics.extend([downlink_pnic_json, uplink_pnic_json])

+

+        return pnics

diff --git a/app/discover/fetchers/aci/aci_fetch_switch_pnic.py b/app/discover/fetchers/aci/aci_fetch_switch_pnic.py
index f0cf6b1..09157b6 100644
--- a/app/discover/fetchers/aci/aci_fetch_switch_pnic.py
+++ b/app/discover/fetchers/aci/aci_fetch_switch_pnic.py
@@ -1,107 +1,108 @@
-###############################################################################
-# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
-# and others                                                                  #
-#                                                                             #
-# All rights reserved. This program and the accompanying materials            #
-# are made available under the terms of the Apache License, Version 2.0       #
-# which accompanies this distribution, and is available at                    #
-# http://www.apache.org/licenses/LICENSE-2.0                                  #
-###############################################################################
-import re
-
-from discover.fetchers.aci.aci_access import AciAccess, aci_config_required
-from utils.inventory_mgr import InventoryMgr
-from utils.util import encode_aci_dn, get_object_path_part
-
-
-# Fetches and adds to database:
-# 1. ACI Switch
-#
-# Returns:
-# 1. ACI Switch pnic that belongs to the ACI Switch (mentioned above)
-# and is connected to Calipso host pnic.
-class AciFetchSwitchPnic(AciAccess):
-
-    def __init__(self):
-        super().__init__()
-        self.inv = InventoryMgr()
-
-    def fetch_pnics_by_mac_address(self, mac_address):
-        mac_filter = "eq(epmMacEp.addr,\"{}\")".format(mac_address)
-        # We are only interested in Ethernet interfaces
-        pnic_filter = "wcard(epmMacEp.ifId, \"eth\")"
-        query_filter = {"query-target-filter":
-                        "and({},{})".format(mac_filter, pnic_filter)}
-
-        pnics = self.fetch_objects_by_class("epmMacEp", query_filter)
-
-        return [pnic["attributes"] for pnic in pnics]
-
-    def fetch_switch_by_id(self, switch_id):
-        dn = "/".join((switch_id, "sys"))
-        response = self.fetch_mo_data(dn)
-        # Unwrap switches
-        switch_data = self.get_objects_by_field_names(response, "topSystem",
-                                                      "attributes")
-        return switch_data[0] if switch_data else None
-
-    @aci_config_required(default=[])
-    def get(self, pnic_id):
-        environment = self.get_env()
-        pnic = self.inv.get_by_id(environment=environment, item_id=pnic_id)
-        if not pnic:
-            return []
-        mac_address = pnic.get("mac_address")
-        if not mac_address:
-            return []
-
-        # Query ACI for related switch pnic
-        leaf_pnics = self.fetch_pnics_by_mac_address(mac_address)
-        if not leaf_pnics:
-            return []
-        leaf_pnic = leaf_pnics[0]
-
-        # Prepare and save switch data in inventory
-        leaf_id_match = re.match("topology/(.+)/sys", leaf_pnic["dn"])
-        if not leaf_id_match:
-            raise ValueError("Failed to fetch leaf switch id from pnic dn: {}"
-                             .format(leaf_pnic["dn"]))
-
-        aci_leaf_id = leaf_id_match.group(1)
-        leaf_data = self.fetch_switch_by_id(aci_leaf_id)
-        if not leaf_data:
-            self.log.warning("No switch found for switch pnic dn: {}"
-                             .format(leaf_pnic["dn"]))
-            return []
-
-        db_leaf_id = "-".join(("switch", encode_aci_dn(aci_leaf_id),
-                               leaf_data["role"]))
-        if not self.inv.get_by_id(environment, db_leaf_id):
-            leaf_json = {
-                "id": db_leaf_id,
-                "ip_address": leaf_data["address"],
-                "type": "switch",
-                "host": db_leaf_id,
-                "aci_document": leaf_data
-            }
-            # Region name is the same as region id
-            region_id = get_object_path_part(pnic["name_path"], "Regions")
-            region = self.inv.get_by_id(environment, region_id)
-            self.inv.save_inventory_object(o=leaf_json, parent=region,
-                                           environment=environment)
-
-        # Prepare pnic json for results list
-        db_pnic_id = "-".join((db_leaf_id,
-                               encode_aci_dn(leaf_pnic["ifId"]),
-                               mac_address))
-        pnic_json = {
-            "id": db_pnic_id,
-            "type": "pnic",
-            "role": "hostlink",
-            "parent_id": db_leaf_id,
-            "pnic_type": "switch",
-            "mac_address": mac_address,
-            "switch": db_leaf_id,
-            "aci_document": leaf_pnic
-        }
-        return [pnic_json]
+###############################################################################

+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #

+# and others                                                                  #

+#                                                                             #

+# All rights reserved. This program and the accompanying materials            #

+# are made available under the terms of the Apache License, Version 2.0       #

+# which accompanies this distribution, and is available at                    #

+# http://www.apache.org/licenses/LICENSE-2.0                                  #

+###############################################################################

+import re

+

+from discover.fetchers.aci.aci_access import AciAccess, aci_config_required

+from utils.inventory_mgr import InventoryMgr

+from utils.util import encode_aci_dn, get_object_path_part

+

+

+# Fetches and adds to database:

+# 1. ACI Switch

+#

+# Returns:

+# 1. ACI Switch pnic that belongs to the ACI Switch (mentioned above)

+# and is connected to Calipso host pnic.

+class AciFetchSwitchPnic(AciAccess):

+

+    def __init__(self):

+        super().__init__()

+        self.inv = InventoryMgr()

+

+    def fetch_pnics_by_mac_address(self, mac_address):

+        mac_filter = "eq(epmMacEp.addr,\"{}\")".format(mac_address)

+        # We are only interested in Ethernet interfaces

+        pnic_filter = "wcard(epmMacEp.ifId, \"eth\")"

+        query_filter = {"query-target-filter":

+                        "and({},{})".format(mac_filter, pnic_filter)}

+

+        pnics = self.fetch_objects_by_class("epmMacEp", query_filter)

+

+        return [pnic["attributes"] for pnic in pnics]

+

+    def fetch_switch_by_id(self, switch_id):

+        dn = "/".join((switch_id, "sys"))

+        response = self.fetch_mo_data(dn)

+        # Unwrap switches

+        switch_data = self.get_objects_by_field_names(response, "topSystem",

+                                                      "attributes")

+        return switch_data[0] if switch_data else None

+

+    @aci_config_required(default=[])

+    def get(self, pnic_id):

+        environment = self.get_env()

+        pnic = self.inv.get_by_id(environment=environment, item_id=pnic_id)

+        if not pnic:

+            return []

+        mac_address = pnic.get("mac_address")

+        if not mac_address:

+            return []

+

+        # Query ACI for related switch pnic

+        leaf_pnics = self.fetch_pnics_by_mac_address(mac_address)

+        if not leaf_pnics:

+            return []

+        leaf_pnic = leaf_pnics[0]

+

+        # Prepare and save switch data in inventory

+        leaf_id_match = re.match("topology/(.+)/sys", leaf_pnic["dn"])

+        if not leaf_id_match:

+            raise ValueError("Failed to fetch leaf switch id from pnic dn: {}"

+                             .format(leaf_pnic["dn"]))

+

+        aci_leaf_id = leaf_id_match.group(1)

+        leaf_data = self.fetch_switch_by_id(aci_leaf_id)

+        if not leaf_data:

+            self.log.warning("No switch found for switch pnic dn: {}"

+                             .format(leaf_pnic["dn"]))

+            return []

+

+        db_leaf_id = "-".join(("switch", encode_aci_dn(aci_leaf_id),

+                               leaf_data["role"]))

+        if not self.inv.get_by_id(environment, db_leaf_id):

+            leaf_json = {

+                "id": db_leaf_id,

+                "ip_address": leaf_data["address"],

+                "type": "switch",

+                "host": db_leaf_id,

+                "aci_document": leaf_data

+            }

+            # Region name is the same as region id

+            region_id = get_object_path_part(pnic["name_path"], "Regions")

+            region = self.inv.get_by_id(environment, region_id)

+            self.inv.save_inventory_object(o=leaf_json, parent=region,

+                                           environment=environment)

+

+        # Prepare pnic json for results list

+        db_pnic_id = "-".join((db_leaf_id,

+                               encode_aci_dn(leaf_pnic["ifId"]),

+                               mac_address))

+        pnic_json = {

+            "id": db_pnic_id,

+            "type": "pnic",

+            "role": "hostlink",

+            "parent_id": db_leaf_id,

+            "parent_type": "switch",

+            "pnic_type": "switch",

+            "mac_address": mac_address,

+            "switch": db_leaf_id,

+            "aci_document": leaf_pnic

+        }

+        return [pnic_json]