diff options
Diffstat (limited to 'app/discover/fetchers')
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_availability_zones.py | 11 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_network.py | 18 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_networks.py | 15 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_port.py | 8 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_ports.py | 8 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_project_hosts.py | 4 | ||||
| -rw-r--r-- | app/discover/fetchers/cli/cli_access.py | 15 | ||||
| -rw-r--r-- | app/discover/fetchers/cli/cli_fetch_host_pnics.py | 7 | ||||
| -rw-r--r-- | app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py | 54 | ||||
| -rw-r--r-- | app/discover/fetchers/cli/cli_fetch_vservice_vnics.py | 10 | ||||
| -rw-r--r-- | app/discover/fetchers/db/db_fetch_oteps.py | 2 |
11 files changed, 78 insertions, 74 deletions
diff --git a/app/discover/fetchers/api/api_fetch_availability_zones.py b/app/discover/fetchers/api/api_fetch_availability_zones.py index 196893b..ad9550e 100644 --- a/app/discover/fetchers/api/api_fetch_availability_zones.py +++ b/app/discover/fetchers/api/api_fetch_availability_zones.py @@ -28,7 +28,7 @@ class ApiFetchAvailabilityZones(ApiAccess): # because the later does not inclde the "internal" zone in the results endpoint = self.get_region_url_nover(region, "nova") req_url = endpoint + "/v2/" + token["tenant"]["id"] + \ - "/os-availability-zone/detail" + "/os-availability-zone/detail" headers = { "X-Auth-Project-Id": project, "X-Auth-Token": token["id"] @@ -45,11 +45,10 @@ class ApiFetchAvailabilityZones(ApiAccess): for doc in azs: doc["id"] = doc["zoneName"] doc["name"] = doc.pop("zoneName") - doc["master_parent_type"] = "region" - doc["master_parent_id"] = region - doc["parent_type"] = "availability_zones_folder" - doc["parent_id"] = region + "-availability_zones" - doc["parent_text"] = "Availability Zones" + self.set_folder_parent(doc, object_type="availability_zone", + master_parent_type="region", + master_parent_id=region, + parent_text="Availability Zones") doc["available"] = doc["zoneState"]["available"] doc.pop("zoneState") ret.append(doc) diff --git a/app/discover/fetchers/api/api_fetch_network.py b/app/discover/fetchers/api/api_fetch_network.py index 889b8a5..b253773 100644 --- a/app/discover/fetchers/api/api_fetch_network.py +++ b/app/discover/fetchers/api/api_fetch_network.py @@ -23,7 +23,8 @@ class ApiFetchNetwork(ApiAccess): return [] ret = [] for region in self.regions: - # TODO: refactor legacy code (Unresolved reference - self.get_for_region) + # TODO: refactor legacy code + # (Unresolved reference - self.get_for_region) ret.extend(self.get_for_region(region, token, project_id)) return ret @@ -37,7 +38,7 @@ class ApiFetchNetwork(ApiAccess): "X-Auth-Token": token["id"] } response = self.get_url(req_url, headers) - if not "network" in response: + if "network" not in response: return [] network = response["network"] subnets = network['subnets'] @@ -60,13 +61,12 @@ class ApiFetchNetwork(ApiAccess): network["cidrs"] = cidrs network["subnet_ids"] = subnet_ids - network["master_parent_type"] = "project" - network["master_parent_id"] = network["tenant_id"] - network["parent_type"] = "networks_folder" - network["parent_id"] = network["tenant_id"] + "-networks" - network["parent_text"] = "Networks" - # set the 'network' attribute for network objects to the name of network, - # to allow setting constraint on network when creating network clique + self.set_folder_parent(network, object_type="network", + master_parent_type="project", + master_parent_id=network["tenant_id"]) + # set the 'network' attribute for network objects to the name of + # network, to allow setting constraint on network when creating + # network clique network['network'] = network["id"] # get the project name project = self.inv.get_by_id(self.get_env(), network["tenant_id"]) diff --git a/app/discover/fetchers/api/api_fetch_networks.py b/app/discover/fetchers/api/api_fetch_networks.py index 4b70f65..f76517a 100644 --- a/app/discover/fetchers/api/api_fetch_networks.py +++ b/app/discover/fetchers/api/api_fetch_networks.py @@ -34,7 +34,7 @@ class ApiFetchNetworks(ApiAccess): "X-Auth-Token": token["id"] } response = self.get_url(req_url, headers) - if not "networks" in response: + if "networks" not in response: return [] networks = response["networks"] req_url = endpoint + "/v2.0/subnets" @@ -46,7 +46,6 @@ class ApiFetchNetworks(ApiAccess): for s in subnets: subnets_hash[s["id"]] = s for doc in networks: - doc["master_parent_type"] = "project" project_id = doc["tenant_id"] if not project_id: # find project ID of admin project @@ -57,12 +56,12 @@ class ApiFetchNetworks(ApiAccess): if not project: self.log.error("failed to find admin project in DB") project_id = project["id"] - doc["master_parent_id"] = project_id - doc["parent_type"] = "networks_folder" - doc["parent_id"] = project_id + "-networks" - doc["parent_text"] = "Networks" - # set the 'network' attribute for network objects to the name of network, - # to allow setting constraint on network when creating network clique + self.set_folder_parent(doc, object_type='network', + master_parent_id=project_id, + master_parent_type='project') + # set the 'network' attribute for network objects to the name of + # network, to allow setting constraint on network when creating + # network clique doc['network'] = doc["id"] # get the project name project = self.inv.get_by_id(self.get_env(), project_id) diff --git a/app/discover/fetchers/api/api_fetch_port.py b/app/discover/fetchers/api/api_fetch_port.py index f8d9eeb..8de1452 100644 --- a/app/discover/fetchers/api/api_fetch_port.py +++ b/app/discover/fetchers/api/api_fetch_port.py @@ -43,11 +43,9 @@ class ApiFetchPort(ApiAccess): return [] doc = response["port"] - doc["master_parent_type"] = "network" - doc["master_parent_id"] = doc["network_id"] - doc["parent_type"] = "ports_folder" - doc["parent_id"] = doc["network_id"] + "-ports" - doc["parent_text"] = "Ports" + self.set_folder_parent(doc, object_type="port", + master_parent_type="network", + master_parent_id=doc["network_id"]) # get the project name net = self.inv.get_by_id(self.get_env(), doc["network_id"]) if net: diff --git a/app/discover/fetchers/api/api_fetch_ports.py b/app/discover/fetchers/api/api_fetch_ports.py index f4c54a6..5e44c1b 100644 --- a/app/discover/fetchers/api/api_fetch_ports.py +++ b/app/discover/fetchers/api/api_fetch_ports.py @@ -38,11 +38,9 @@ class ApiFetchPorts(ApiAccess): return [] ports = response["ports"] for doc in ports: - doc["master_parent_type"] = "network" - doc["master_parent_id"] = doc["network_id"] - doc["parent_type"] = "ports_folder" - doc["parent_id"] = doc["network_id"] + "-ports" - doc["parent_text"] = "Ports" + self.set_folder_parent(doc, object_type="port", + master_parent_type="network", + master_parent_id=doc["network_id"]) # get the project name net = self.inv.get_by_id(self.get_env(), doc["network_id"]) if net: diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py index 2aeb24f..1059600 100644 --- a/app/discover/fetchers/api/api_fetch_project_hosts.py +++ b/app/discover/fetchers/api/api_fetch_project_hosts.py @@ -11,11 +11,11 @@ import json from discover.fetchers.api.api_access import ApiAccess from discover.fetchers.db.db_access import DbAccess -from discover.fetchers.cli.cli_access import CliAccess +from discover.fetchers.cli.cli_fetch_host_details import CliFetchHostDetails from utils.ssh_connection import SshError -class ApiFetchProjectHosts(ApiAccess, DbAccess, CliAccess): +class ApiFetchProjectHosts(ApiAccess, DbAccess, CliFetchHostDetails): def __init__(self): super(ApiFetchProjectHosts, self).__init__() diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py index c77b22a..68b81c8 100644 --- a/app/discover/fetchers/cli/cli_access.py +++ b/app/discover/fetchers/cli/cli_access.py @@ -17,7 +17,7 @@ from utils.logging.console_logger import ConsoleLogger from utils.ssh_conn import SshConn -class CliAccess(BinaryConverter, Fetcher): +class CliAccess(Fetcher, BinaryConverter): connections = {} ssh_cmd = "ssh -q -o StrictHostKeyChecking=no " call_count_per_con = {} @@ -71,8 +71,9 @@ class CliAccess(BinaryConverter, Fetcher): self.cached_commands[cmd_path] = {"timestamp": curr_time, "result": ret} return ret - def run_fetch_lines(self, cmd, ssh_to_host="", enable_cache=True): - out = self.run(cmd, ssh_to_host, enable_cache) + def run_fetch_lines(self, cmd, ssh_to_host="", enable_cache=True, + use_sudo=True): + out = self.run(cmd, ssh_to_host, enable_cache, use_sudo=use_sudo) if not out: return [] # first try to split lines by whitespace @@ -236,7 +237,7 @@ class CliAccess(BinaryConverter, Fetcher): self.find_matching_regexps(o, line, regexps) for regexp_tuple in regexps: name = regexp_tuple['name'] - if 'name' not in o and 'default' in regexp_tuple: + if name not in o and 'default' in regexp_tuple: o[name] = regexp_tuple['default'] @staticmethod @@ -247,4 +248,8 @@ class CliAccess(BinaryConverter, Fetcher): regex = re.compile(regex) matches = regex.search(line) if matches and name not in o: - o[name] = matches.group(1) + try: + o[name] = matches.group(1) + except IndexError as e: + self.log.error('failed to find group 1 in match, {}' + .format(str(regexp_tuple))) diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py index 26cd603..81d164d 100644 --- a/app/discover/fetchers/cli/cli_fetch_host_pnics.py +++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py @@ -27,8 +27,8 @@ class CliFetchHostPnics(CliAccess): 'description': 'IPv6 Address'} ] - def get(self, id): - host_id = id[:id.rindex("-")] + def get(self, parent_id): + host_id = parent_id[:parent_id.rindex("-")] cmd = 'ls -l /sys/class/net | grep ^l | grep -v "/virtual/"' host = self.inv.get_by_id(self.get_env(), host_id) if not host: @@ -39,7 +39,8 @@ class CliFetchHostPnics(CliAccess): ", host: " + str(host)) return [] host_types = host["host_type"] - if "Network" not in host_types and "Compute" not in host_types: + accepted_host_types = ['Network', 'Compute', 'Kube-node'] + if not [t for t in accepted_host_types if t in host_types]: return [] interface_lines = self.run_fetch_lines(cmd, host_id) interfaces = [] diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py index ff37569..ac04568 100644 --- a/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py @@ -18,8 +18,8 @@ class CliFetchVconnectorsOvs(CliFetchVconnectors): def get_vconnectors(self, host): host_id = host['id'] - lines = self.run_fetch_lines("brctl show", host_id) - headers = ["bridge_name", "bridge_id", "stp_enabled", "interfaces"] + lines = self.run_fetch_lines('brctl show', host_id) + headers = ['bridge_name', 'bridge_id', 'stp_enabled', 'interfaces'] headers_count = len(headers) # since we hard-coded the headers list, remove the headers line del lines[:1] @@ -31,26 +31,32 @@ class CliFetchVconnectorsOvs(CliFetchVconnectors): results = self.parse_cmd_result_with_whitespace(fixed_lines, headers, False) ret = [] for doc in results: - doc["name"] = doc.pop("bridge_name") - doc["id"] = doc["name"] + "-" + doc.pop("bridge_id") - doc["host"] = host_id - doc["connector_type"] = "bridge" - if "interfaces" in doc: - interfaces = {} - interface_names = doc["interfaces"].split(",") - for interface_name in interface_names: - # find MAC address for this interface from ports list - port_id_prefix = interface_name[3:] - port = self.inv.find_items({ - "environment": self.get_env(), - "type": "port", - "binding:host_id": host_id, - "id": {"$regex": r"^" + re.escape(port_id_prefix)} - }, get_single=True) - mac_address = '' if not port else port['mac_address'] - interface = {'name': interface_name, 'mac_address': mac_address} - interfaces[interface_name] = interface - doc["interfaces"] = interfaces - doc['interfaces_names'] = list(interfaces.keys()) - ret.append(doc) + doc['name'] = '{}-{}'.format(host_id, doc['bridge_name']) + doc['id'] = '{}-{}'.format(doc['name'], doc.pop('bridge_id')) + doc['host'] = host_id + doc['connector_type'] = 'bridge' + self.get_vconnector_interfaces(doc, host_id) + ret.append(doc) return ret + + def get_vconnector_interfaces(self, doc, host_id): + if 'interfaces' not in doc: + doc['interfaces'] = {} + doc['interfaces_names'] = [] + return + interfaces = {} + interface_names = doc['interfaces'].split(',') + for interface_name in interface_names: + # find MAC address for this interface from ports list + port_id_prefix = interface_name[3:] + port = self.inv.find_items({ + 'environment': self.get_env(), + 'type': 'port', + 'binding:host_id': host_id, + 'id': {'$regex': r'^' + re.escape(port_id_prefix)} + }, get_single=True) + mac_address = '' if not port else port['mac_address'] + interface = {'name': interface_name, 'mac_address': mac_address} + interfaces[interface_name] = interface + doc['interfaces'] = interfaces + doc['interfaces_names'] = list(interfaces.keys()) diff --git a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py index 3bc3a5b..0129d3b 100644 --- a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py +++ b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py @@ -66,17 +66,15 @@ class CliFetchVserviceVnics(CliAccess): master_parent_id = "{}-{}".format(host, service) current = { "id": host + "-" + name, - "type": "vnic", "vnic_type": "vservice_vnic", "host": host, "name": name, - "master_parent_type": "vservice", - "master_parent_id": master_parent_id, - "parent_type": "vnics_folder", - "parent_id": "{}-vnics".format(master_parent_id), - "parent_text": "vNICs", "lines": [] } + self.set_folder_parent(current, object_type="vnic", + master_parent_type="vservice", + master_parent_id=master_parent_id, + parent_text="vNICs") interfaces.append(current) self.handle_line(current, line_remainder) else: diff --git a/app/discover/fetchers/db/db_fetch_oteps.py b/app/discover/fetchers/db/db_fetch_oteps.py index 85376ed..7721136 100644 --- a/app/discover/fetchers/db/db_fetch_oteps.py +++ b/app/discover/fetchers/db/db_fetch_oteps.py @@ -82,4 +82,4 @@ class DbFetchOteps(DbAccess, CliAccess, metaclass=Singleton): interface = l.split(":")[1].strip() if vconnector: - doc["vconnector"] = vconnector + doc["vconnector"] = '{}-{}'.format(host_id, vconnector) |