diff options
author | 2017-07-27 15:04:07 +0000 | |
---|---|---|
committer | 2017-07-27 15:04:07 +0000 | |
commit | 162c03ef301396072c1934e7e7e0c40a841b4fe2 (patch) | |
tree | 7a2a2781949252436450ff5832962785061a774a /app/discover/fetchers | |
parent | b88c78e3cf2bef22aa2f1c4d0bf305e303bc15f0 (diff) | |
parent | 7e83d0876ddb84a45e130eeba28bc40ef53c074b (diff) |
Merge "Calipso initial release for OPNFV"
Diffstat (limited to 'app/discover/fetchers')
46 files changed, 3032 insertions, 0 deletions
diff --git a/app/discover/fetchers/__init__.py b/app/discover/fetchers/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/aci/__init__.py b/app/discover/fetchers/aci/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/aci/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/aci/aci_access.py b/app/discover/fetchers/aci/aci_access.py new file mode 100644 index 0000000..836e45d --- /dev/null +++ b/app/discover/fetchers/aci/aci_access.py @@ -0,0 +1,200 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +import requests + +from discover.configuration import Configuration +from discover.fetcher import Fetcher + + +def aci_config_required(default=None): + def decorator(func): + def wrapper(self, *args, **kwargs): + if not self.aci_enabled: + return default + return func(self, *args, **kwargs) + return wrapper + return decorator + + +class AciAccess(Fetcher): + + RESPONSE_FORMAT = "json" + cookie_token = None + + def __init__(self): + super().__init__() + self.configuration = Configuration() + self.aci_enabled = self.configuration.get_env_config() \ + .get('aci_enabled', False) + self.aci_configuration = None + self.host = None + if self.aci_enabled: + self.aci_configuration = self.configuration.get("ACI") + self.host = self.aci_configuration["host"] + + def get_base_url(self): + return "https://{}/api".format(self.host) + + # Unwrap ACI response payload + # and return an array of desired fields' values. + # + # Parameters + # ---------- + # + # payload: dict + # Full json response payload returned by ACI + # *field_names: Tuple[str] + # Enumeration of fields that are used to traverse ACI "imdata" array + # (order is important) + # + # Returns + # ---------- + # list + # List of unwrapped dictionaries (or primitives) + # + # Example + # ---------- + # Given payload: + # + # { + # "totalCount": "2", + # "imdata": [ + # { + # "aaa": { + # "bbb": { + # "ccc": "value1" + # } + # } + # }, + # { + # "aaa": { + # "bbb": { + # "ccc": "value2" + # } + # } + # } + # ] + # } + # + # Executing get_objects_by_field_names(payload, "aaa", "bbb") + # will yield the following result: + # + # >>> [{"ccc": "value1"}, {"ccc": "value2"}] + # + # Executing get_objects_by_field_names(payload, "aaa", "bbb", "ccc") + # will yield the following result: + # + # >>> ["value1", "value2"] + # + @staticmethod + def get_objects_by_field_names(payload, *field_names): + results = payload.get("imdata", []) + if not results: + return [] + + for field in field_names: + results = [entry[field] for entry in results] + return results + + # Set auth tokens in request headers and cookies + @staticmethod + def _insert_token_into_request(cookies): + return dict(cookies, **AciAccess.cookie_token) \ + if cookies \ + else AciAccess.cookie_token + + @staticmethod + def _set_token(response): + tokens = AciAccess.get_objects_by_field_names(response.json(), "aaaLogin", "attributes", "token") + token = tokens[0] + + AciAccess.cookie_token = {"APIC-Cookie": token} + + @aci_config_required() + def login(self): + url = "/".join((self.get_base_url(), "aaaLogin.json")) + payload = { + "aaaUser": { + "attributes": { + "name": self.aci_configuration["user"], + "pwd": self.aci_configuration["pwd"] + } + } + } + + response = requests.post(url, json=payload, verify=False) + response.raise_for_status() + + AciAccess._set_token(response) + + # Refresh token or login if token has expired + @aci_config_required() + def refresh_token(self): + # First time login + if not AciAccess.cookie_token: + self.login() + return + + url = "/".join((self.get_base_url(), "aaaRefresh.json")) + + response = requests.get(url, verify=False) + + # Login again if the token has expired + if response.status_code == requests.codes.forbidden: + self.login() + return + # Propagate any other error + elif response.status_code != requests.codes.ok: + response.raise_for_status() + + AciAccess._set_token(response) + + @aci_config_required(default={}) + def send_get(self, url, params, headers, cookies): + self.refresh_token() + + cookies = self._insert_token_into_request(cookies) + + response = requests.get(url, params=params, headers=headers, + cookies=cookies, verify=False) + # Let client handle HTTP errors + response.raise_for_status() + + return response.json() + + # Search ACI for Managed Objects (MOs) of a specific class + @aci_config_required(default=[]) + def fetch_objects_by_class(self, + class_name: str, + params: dict = None, + headers: dict = None, + cookies: dict = None, + response_format: str = RESPONSE_FORMAT): + url = "/".join((self.get_base_url(), + "class", "{cn}.{f}".format(cn=class_name, f=response_format))) + + response_json = self.send_get(url, params, headers, cookies) + return self.get_objects_by_field_names(response_json, class_name) + + # Fetch data for a specific Managed Object (MO) + @aci_config_required(default=[]) + def fetch_mo_data(self, + dn: str, + params: dict = None, + headers: dict = None, + cookies: dict = None, + response_format: str = RESPONSE_FORMAT): + url = "/".join((self.get_base_url(), "mo", "topology", + "{dn}.{f}".format(dn=dn, f=response_format))) + + response_json = self.send_get(url, params, headers, cookies) + return response_json diff --git a/app/discover/fetchers/aci/aci_fetch_switch_pnic.py b/app/discover/fetchers/aci/aci_fetch_switch_pnic.py new file mode 100644 index 0000000..a4216ea --- /dev/null +++ b/app/discover/fetchers/aci/aci_fetch_switch_pnic.py @@ -0,0 +1,91 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.aci.aci_access import AciAccess, aci_config_required +from utils.inventory_mgr import InventoryMgr +from utils.util import encode_aci_dn, get_object_path_part + + +class AciFetchSwitchPnic(AciAccess): + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def fetch_pnics_by_mac_address(self, mac_address): + mac_filter = "eq(epmMacEp.addr,\"{}\")".format(mac_address) + pnic_filter = "wcard(epmMacEp.ifId, \"eth\")" + query_filter = "and({},{})".format(mac_filter, pnic_filter) + + pnics = self.fetch_objects_by_class("epmMacEp", + {"query-target-filter": query_filter}) + + return [pnic["attributes"] for pnic in pnics] + + def fetch_switch_by_id(self, switch_id): + dn = "/".join((switch_id, "sys")) + response = self.fetch_mo_data(dn) + switch_data = self.get_objects_by_field_names(response, "topSystem", "attributes") + return switch_data[0] if switch_data else None + + @aci_config_required(default=[]) + def get(self, pnic_id): + environment = self.get_env() + pnic = self.inv.get_by_id(environment=environment, item_id=pnic_id) + if not pnic: + return [] + mac_address = pnic.get("mac_address") + if not mac_address: + return [] + + switch_pnics = self.fetch_pnics_by_mac_address(mac_address) + if not switch_pnics: + return [] + switch_pnic = switch_pnics[0] + + # Prepare and save switch data in inventory + aci_id_match = re.match("topology/(.+)/sys", switch_pnic["dn"]) + if not aci_id_match: + raise ValueError("Failed to fetch switch id from pnic dn: {}" + .format(switch_pnic["dn"])) + + aci_switch_id = aci_id_match.group(1) + db_switch_id = encode_aci_dn(aci_switch_id) + if not self.inv.get_by_id(environment, db_switch_id): + switch_data = self.fetch_switch_by_id(aci_switch_id) + if not switch_data: + self.log.warning("No switch found for switch pnic dn: {}" + .format(switch_pnic["dn"])) + return [] + + switch_json = { + "id": db_switch_id, + "ip_address": switch_data["address"], + "type": "switch", + "aci_document": switch_data + } + # Region name is the same as region id + region_id = get_object_path_part(pnic["name_path"], "Regions") + region = self.inv.get_by_id(environment, region_id) + self.inv.save_inventory_object(o=switch_json, parent=region, environment=environment) + + db_pnic_id = "-".join((db_switch_id, + encode_aci_dn(switch_pnic["ifId"]), + mac_address)) + pnic_json = { + "id": db_pnic_id, + "type": "pnic", + "pnic_type": "switch", + "mac_address": mac_address, + "aci_document": switch_pnic + } + return [pnic_json] + diff --git a/app/discover/fetchers/api/__init__.py b/app/discover/fetchers/api/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/api/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/api/api_access.py b/app/discover/fetchers/api/api_access.py new file mode 100644 index 0000000..89eeb34 --- /dev/null +++ b/app/discover/fetchers/api/api_access.py @@ -0,0 +1,195 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import calendar +import re +import requests +import time + +from discover.configuration import Configuration +from discover.fetcher import Fetcher +from utils.string_utils import jsonify + + +class ApiAccess(Fetcher): + subject_token = None + initialized = False + regions = {} + config = None + api_config = None + + host = "" + base_url = "" + admin_token = "" + tokens = {} + admin_endpoint = "" + admin_project = None + auth_response = None + + alternative_services = { + "neutron": ["quantum"] + } + + # identitity API v2 version with admin token + def __init__(self): + super(ApiAccess, self).__init__() + if ApiAccess.initialized: + return + ApiAccess.config = Configuration() + ApiAccess.api_config = ApiAccess.config.get("OpenStack") + host = ApiAccess.api_config["host"] + ApiAccess.host = host + port = ApiAccess.api_config["port"] + if not (host and port): + raise ValueError('Missing definition of host or port ' + + 'for OpenStack API access') + ApiAccess.base_url = "http://" + host + ":" + port + ApiAccess.admin_token = ApiAccess.api_config["admin_token"] + ApiAccess.admin_project = ApiAccess.api_config["admin_project"] \ + if "admin_project" in ApiAccess.api_config \ + else 'admin' + ApiAccess.admin_endpoint = "http://" + host + ":" + "35357" + + token = self.v2_auth_pwd(ApiAccess.admin_project) + if not token: + raise ValueError("Authentication failed. Failed to obtain token") + else: + self.subject_token = token + + @staticmethod + def parse_time(time_str): + try: + time_struct = time.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + try: + time_struct = time.strptime(time_str, + "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + return None + return time_struct + + # try to use existing token, if it did not expire + def get_existing_token(self, project_id): + try: + token_details = ApiAccess.tokens[project_id] + except KeyError: + return None + token_expiry = token_details["expires"] + token_expiry_time_struct = self.parse_time(token_expiry) + if not token_expiry_time_struct: + return None + token_expiry_time = token_details["token_expiry_time"] + now = time.time() + if now > token_expiry_time: + # token has expired + ApiAccess.tokens.pop(project_id) + return None + return token_details + + def v2_auth(self, project_id, headers, post_body): + subject_token = self.get_existing_token(project_id) + if subject_token: + return subject_token + req_url = ApiAccess.base_url + "/v2.0/tokens" + response = requests.post(req_url, json=post_body, headers=headers) + ApiAccess.auth_response = response.json() + if 'error' in self.auth_response: + e = self.auth_response['error'] + self.log.error(str(e['code']) + ' ' + e['title'] + ': ' + + e['message'] + ", URL: " + req_url) + return None + try: + token_details = ApiAccess.auth_response["access"]["token"] + except KeyError: + # assume authentication failed + return None + token_expiry = token_details["expires"] + token_expiry_time_struct = self.parse_time(token_expiry) + if not token_expiry_time_struct: + return None + token_expiry_time = calendar.timegm(token_expiry_time_struct) + token_details["token_expiry_time"] = token_expiry_time + ApiAccess.tokens[project_id] = token_details + return token_details + + def v2_auth_pwd(self, project): + user = ApiAccess.api_config["user"] + pwd = ApiAccess.api_config["pwd"] + post_body = { + "auth": { + "passwordCredentials": { + "username": user, + "password": pwd + } + } + } + if project is not None: + post_body["auth"]["tenantName"] = project + project_id = project + else: + project_id = "" + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json; charset=UTF-8' + } + return self.v2_auth(project_id, headers, post_body) + + def get_rel_url(self, relative_url, headers): + req_url = ApiAccess.base_url + relative_url + return self.get_url(req_url, headers) + + def get_url(self, req_url, headers): + response = requests.get(req_url, headers=headers) + if response.status_code != requests.codes.ok: + # some error happened + if "reason" in response: + msg = ", reason: {}".format(response.reason) + else: + msg = ", response: {}".format(response.text) + self.log.error("req_url: {} {}".format(req_url, msg)) + return response + ret = response.json() + return ret + + def get_region_url(self, region_name, service): + if region_name not in self.regions: + return None + region = self.regions[region_name] + s = self.get_service_region_endpoints(region, service) + if not s: + return None + orig_url = s["adminURL"] + # replace host name with the host found in config + url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url) + return url + + # like get_region_url(), but remove everything starting from the "/v2" + def get_region_url_nover(self, region, service): + full_url = self.get_region_url(region, service) + if not full_url: + self.log.error("could not find region URL for region: " + region) + exit() + url = re.sub(r":([0-9]+)/v[2-9].*", r":\1", full_url) + return url + + def get_catalog(self, pretty): + return jsonify(self.regions, pretty) + + # find the endpoints for a given service name, + # considering also alternative service names + def get_service_region_endpoints(self, region, service): + alternatives = [service] + endpoints = region["endpoints"] + if service in self.alternative_services: + alternatives.extend(self.alternative_services[service]) + for sname in alternatives: + if sname in endpoints: + return endpoints[sname] + return None + diff --git a/app/discover/fetchers/api/api_fetch_availability_zones.py b/app/discover/fetchers/api/api_fetch_availability_zones.py new file mode 100644 index 0000000..196893b --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_availability_zones.py @@ -0,0 +1,56 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess + + +class ApiFetchAvailabilityZones(ApiAccess): + def __init__(self): + super(ApiFetchAvailabilityZones, self).__init__() + + def get(self, project_id): + token = self.v2_auth_pwd(project_id) + if not token: + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_for_region(project_id, region, token)) + return ret + + def get_for_region(self, project, region, token): + # we use os-availability-zone/detail rather than os-availability-zone, + # because the later does not inclde the "internal" zone in the results + endpoint = self.get_region_url_nover(region, "nova") + req_url = endpoint + "/v2/" + token["tenant"]["id"] + \ + "/os-availability-zone/detail" + headers = { + "X-Auth-Project-Id": project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if "status" in response and int(response["status"]) != 200: + return [] + ret = [] + if "availabilityZoneInfo" not in response: + return [] + azs = response["availabilityZoneInfo"] + if not azs: + return [] + for doc in azs: + doc["id"] = doc["zoneName"] + doc["name"] = doc.pop("zoneName") + doc["master_parent_type"] = "region" + doc["master_parent_id"] = region + doc["parent_type"] = "availability_zones_folder" + doc["parent_id"] = region + "-availability_zones" + doc["parent_text"] = "Availability Zones" + doc["available"] = doc["zoneState"]["available"] + doc.pop("zoneState") + ret.append(doc) + return ret diff --git a/app/discover/fetchers/api/api_fetch_end_points.py b/app/discover/fetchers/api/api_fetch_end_points.py new file mode 100644 index 0000000..9471c7e --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_end_points.py @@ -0,0 +1,35 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# fetch the end points for a given project (tenant) +# return list of regions, to allow further recursive scanning + +from discover.fetchers.api.api_access import ApiAccess + + +class ApiFetchEndPoints(ApiAccess): + + def get(self, project_id): + if project_id != "admin": + return [] # XXX currently having problems authenticating to other tenants + self.v2_auth_pwd(project_id) + + environment = ApiAccess.config.get_env_name() + regions = [] + services = ApiAccess.auth_response['access']['serviceCatalog'] + endpoints = [] + for s in services: + if s["type"] != "identity": + continue + e = s["endpoints"][0] + e["environment"] = environment + e["project"] = project_id + e["type"] = "endpoint" + endpoints.append(e) + return endpoints diff --git a/app/discover/fetchers/api/api_fetch_host_instances.py b/app/discover/fetchers/api/api_fetch_host_instances.py new file mode 100644 index 0000000..56cffda --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_host_instances.py @@ -0,0 +1,59 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from discover.fetchers.db.db_access import DbAccess +from discover.fetchers.db.db_fetch_instances import DbFetchInstances +from utils.inventory_mgr import InventoryMgr +from utils.singleton import Singleton + + +class ApiFetchHostInstances(ApiAccess, DbAccess, metaclass=Singleton): + def __init__(self): + super(ApiFetchHostInstances, self).__init__() + self.inv = InventoryMgr() + self.endpoint = ApiAccess.base_url.replace(":5000", ":8774") + self.projects = None + self.db_fetcher = DbFetchInstances() + + def get_projects(self): + if not self.projects: + projects_list = self.inv.get(self.get_env(), "project", None) + self.projects = [p["name"] for p in projects_list] + + def get(self, id): + self.get_projects() + host_id = id[:id.rindex("-")] + host = self.inv.get_by_id(self.get_env(), host_id) + if not host or "Compute" not in host.get("host_type", ""): + return [] + instances_found = self.get_instances_from_api(host_id) + self.db_fetcher.get_instance_data(instances_found) + return instances_found + + def get_instances_from_api(self, host_name): + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + tenant_id = token["tenant"]["id"] + req_url = self.endpoint + "/v2/" + tenant_id + \ + "/os-hypervisors/" + host_name + "/servers" + response = self.get_url(req_url, {"X-Auth-Token": token["id"]}) + ret = [] + if not "hypervisors" in response: + return [] + if not "servers" in response["hypervisors"][0]: + return [] + for doc in response["hypervisors"][0]["servers"]: + doc["id"] = doc["uuid"] + doc["host"] = host_name + doc["local_name"] = doc.pop("name") + ret.append(doc) + self.log.info("found %s instances for host: %s", str(len(ret)), host_name) + return ret diff --git a/app/discover/fetchers/api/api_fetch_network.py b/app/discover/fetchers/api/api_fetch_network.py new file mode 100644 index 0000000..889b8a5 --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_network.py @@ -0,0 +1,76 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from utils.inventory_mgr import InventoryMgr + + +class ApiFetchNetwork(ApiAccess): + def __init__(self): + super(ApiFetchNetwork, self).__init__() + self.inv = InventoryMgr() + + def get(self, project_id): + # use project admin credentials, to be able to fetch all networks + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + # TODO: refactor legacy code (Unresolved reference - self.get_for_region) + ret.extend(self.get_for_region(region, token, project_id)) + return ret + + def get_network(self, region, token, subnet_id): + endpoint = self.get_region_url_nover(region, "neutron") + + # get target network network document + req_url = endpoint + "/v2.0/networks/" + subnet_id + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not "network" in response: + return [] + network = response["network"] + subnets = network['subnets'] + + # get subnets documents. + subnets_hash = {} + cidrs = [] + subnet_ids = [] + for subnet_id in subnets: + req_url = endpoint + "/v2.0/subnets/" + subnet_id + response = self.get_url(req_url, headers) + if "subnet" in response: + # create a hash subnets, to allow easy locating of subnets + subnet = response["subnet"] + subnets_hash[subnet["name"]] = subnet + cidrs.append(subnet["cidr"]) + subnet_ids.append(subnet["id"]) + + network["subnets"] = subnets_hash + network["cidrs"] = cidrs + network["subnet_ids"] = subnet_ids + + network["master_parent_type"] = "project" + network["master_parent_id"] = network["tenant_id"] + network["parent_type"] = "networks_folder" + network["parent_id"] = network["tenant_id"] + "-networks" + network["parent_text"] = "Networks" + # set the 'network' attribute for network objects to the name of network, + # to allow setting constraint on network when creating network clique + network['network'] = network["id"] + # get the project name + project = self.inv.get_by_id(self.get_env(), network["tenant_id"]) + if project: + network["project"] = project["name"] + + return network diff --git a/app/discover/fetchers/api/api_fetch_networks.py b/app/discover/fetchers/api/api_fetch_networks.py new file mode 100644 index 0000000..4b70f65 --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_networks.py @@ -0,0 +1,86 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from utils.inventory_mgr import InventoryMgr + + +class ApiFetchNetworks(ApiAccess): + def __init__(self): + super(ApiFetchNetworks, self).__init__() + self.inv = InventoryMgr() + + def get(self, project_id=None): + # use project admin credentials, to be able to fetch all networks + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_networks(region, token)) + return ret + + def get_networks(self, region, token): + endpoint = self.get_region_url_nover(region, "neutron") + req_url = endpoint + "/v2.0/networks" + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not "networks" in response: + return [] + networks = response["networks"] + req_url = endpoint + "/v2.0/subnets" + response = self.get_url(req_url, headers) + subnets_hash = {} + if "subnets" in response: + # create a hash subnets, to allow easy locating of subnets + subnets = response["subnets"] + for s in subnets: + subnets_hash[s["id"]] = s + for doc in networks: + doc["master_parent_type"] = "project" + project_id = doc["tenant_id"] + if not project_id: + # find project ID of admin project + project = self.inv.get_by_field(self.get_env(), + "project", "name", + self.admin_project, + get_single=True) + if not project: + self.log.error("failed to find admin project in DB") + project_id = project["id"] + doc["master_parent_id"] = project_id + doc["parent_type"] = "networks_folder" + doc["parent_id"] = project_id + "-networks" + doc["parent_text"] = "Networks" + # set the 'network' attribute for network objects to the name of network, + # to allow setting constraint on network when creating network clique + doc['network'] = doc["id"] + # get the project name + project = self.inv.get_by_id(self.get_env(), project_id) + if project: + doc["project"] = project["name"] + subnets_details = {} + cidrs = [] + subnet_ids = [] + for s in doc["subnets"]: + try: + subnet = subnets_hash[s] + cidrs.append(subnet["cidr"]) + subnet_ids.append(subnet["id"]) + subnets_details[subnet["name"]] = subnet + except KeyError: + pass + + doc["subnets"] = subnets_details + doc["cidrs"] = cidrs + doc["subnet_ids"] = subnet_ids + return networks diff --git a/app/discover/fetchers/api/api_fetch_port.py b/app/discover/fetchers/api/api_fetch_port.py new file mode 100644 index 0000000..f8d9eeb --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_port.py @@ -0,0 +1,60 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from utils.inventory_mgr import InventoryMgr + + +class ApiFetchPort(ApiAccess): + def __init__(self): + super(ApiFetchPort, self).__init__() + self.inv = InventoryMgr() + + def get(self, project_id): + if not project_id: + self.log.info("Get method needs ID parameter") + return [] + # use project admin credentials, to be able to fetch all ports + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + ret.append(self.get_port(region, token, project_id)) + if ret == []: + self.log.info("ApiFetchPort: Port not found.") + return ret + + def get_port(self, region, token, id): + endpoint = self.get_region_url_nover(region, "neutron") + req_url = endpoint + "/v2.0/ports/" + id + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not "port" in response: + return [] + + doc = response["port"] + doc["master_parent_type"] = "network" + doc["master_parent_id"] = doc["network_id"] + doc["parent_type"] = "ports_folder" + doc["parent_id"] = doc["network_id"] + "-ports" + doc["parent_text"] = "Ports" + # get the project name + net = self.inv.get_by_id(self.get_env(), doc["network_id"]) + if net: + doc["name"] = doc["mac_address"] + else: + doc["name"] = doc["id"] + project = self.inv.get_by_id(self.get_env(), doc["tenant_id"]) + if project: + doc["project"] = project["name"] + return doc diff --git a/app/discover/fetchers/api/api_fetch_ports.py b/app/discover/fetchers/api/api_fetch_ports.py new file mode 100644 index 0000000..f4c54a6 --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_ports.py @@ -0,0 +1,55 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from utils.inventory_mgr import InventoryMgr + + +class ApiFetchPorts(ApiAccess): + def __init__(self): + super(ApiFetchPorts, self).__init__() + self.inv = InventoryMgr() + + def get(self, project_id): + # use project admin credentials, to be able to fetch all ports + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_ports_for_region(region, token)) + return ret + + def get_ports_for_region(self, region, token): + endpoint = self.get_region_url_nover(region, "neutron") + req_url = endpoint + "/v2.0/ports" + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not "ports" in response: + return [] + ports = response["ports"] + for doc in ports: + doc["master_parent_type"] = "network" + doc["master_parent_id"] = doc["network_id"] + doc["parent_type"] = "ports_folder" + doc["parent_id"] = doc["network_id"] + "-ports" + doc["parent_text"] = "Ports" + # get the project name + net = self.inv.get_by_id(self.get_env(), doc["network_id"]) + if net: + doc["name"] = doc["mac_address"] + else: + doc["name"] = doc["id"] + project = self.inv.get_by_id(self.get_env(), doc["tenant_id"]) + if project: + doc["project"] = project["name"] + return ports diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py new file mode 100644 index 0000000..7dc262e --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_project_hosts.py @@ -0,0 +1,144 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +from discover.fetchers.api.api_access import ApiAccess +from discover.fetchers.db.db_access import DbAccess + + +class ApiFetchProjectHosts(ApiAccess, DbAccess): + def __init__(self): + super(ApiFetchProjectHosts, self).__init__() + + def get(self, project_id): + if project_id != self.admin_project: + # do not scan hosts except under project 'admin' + return [] + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_for_region(region, token)) + return ret + + def get_for_region(self, region, token): + endpoint = self.get_region_url(region, "nova") + ret = [] + if not token: + return [] + req_url = endpoint + "/os-availability-zone/detail" + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if "status" in response and int(response["status"]) != 200: + return [] + az_info = response["availabilityZoneInfo"] + hosts = {} + for doc in az_info: + az_hosts = self.get_hosts_from_az(doc) + for h in az_hosts: + if h["name"] in hosts: + # merge host_type data between AZs + existing_entry = hosts[h["name"]] + for t in h["host_type"]: + self.add_host_type(existing_entry, t, doc['zoneName']) + else: + hosts[h["name"]] = h + ret.append(h) + # get os_id for hosts using the os-hypervisors API call + req_url = endpoint + "/os-hypervisors" + response = self.get_url(req_url, headers) + if "status" in response and int(response["status"]) != 200: + return ret + if "hypervisors" not in response: + return ret + for h in response["hypervisors"]: + hvname = h["hypervisor_hostname"] + if '.' in hvname and hvname not in hosts: + hostname = hvname[:hvname.index('.')] + else: + hostname = hvname + try: + doc = hosts[hostname] + except KeyError: + # TBD - add error output + continue + doc["os_id"] = str(h["id"]) + self.fetch_compute_node_ip_address(doc, hvname) + # get more network nodes details + self.fetch_network_node_details(ret) + return ret + + def get_hosts_from_az(self, az): + ret = [] + for h in az["hosts"]: + doc = self.get_host_details(az, h) + ret.append(doc) + return ret + + def get_host_details(self, az, h): + # for hosts we use the name + services = az["hosts"][h] + doc = { + "id": h, + "host": h, + "name": h, + "zone": az["zoneName"], + "parent_type": "availability_zone", + "parent_id": az["zoneName"], + "services": services, + "host_type": [] + } + if "nova-conductor" in services: + s = services["nova-conductor"] + if s["available"] and s["active"]: + self.add_host_type(doc, "Controller", az['zoneName']) + if "nova-compute" in services: + s = services["nova-compute"] + if s["available"] and s["active"]: + self.add_host_type(doc, "Compute", az['zoneName']) + return doc + + # fetch more details of network nodes from neutron.agents table + def fetch_network_node_details(self, docs): + hosts = {} + for doc in docs: + hosts[doc["host"]] = doc + query = """ + SELECT DISTINCT host, host AS id, configurations + FROM {}.agents + WHERE agent_type IN ('Metadata agent', 'DHCP agent', 'L3 agent') + """.format(self.neutron_db) + results = self.get_objects_list(query, "") + for r in results: + host = hosts[r["host"]] + host["config"] = json.loads(r["configurations"]) + self.add_host_type(host, "Network", '') + + # fetch ip_address from nova.compute_nodes table if possible + def fetch_compute_node_ip_address(self, doc, h): + query = """ + SELECT host_ip AS ip_address + FROM nova.compute_nodes + WHERE hypervisor_hostname = %s + """ + results = self.get_objects_list_for_id(query, "", h) + for db_row in results: + doc.update(db_row) + + def add_host_type(self, doc, type, zone): + if not type in doc["host_type"]: + doc["host_type"].append(type) + if type == 'Compute': + doc['zone'] = zone + doc['parent_id'] = zone diff --git a/app/discover/fetchers/api/api_fetch_projects.py b/app/discover/fetchers/api/api_fetch_projects.py new file mode 100644 index 0000000..4ef8083 --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_projects.py @@ -0,0 +1,66 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess + + +class ApiFetchProjects(ApiAccess): + def __init__(self): + super(ApiFetchProjects, self).__init__() + + def get(self, project_id): + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + if not self.regions: + self.log.error('No regions found') + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_for_region(region, token)) + projects_for_user = self.get_projects_for_api_user(region, token) + return [p for p in ret if p['name'] in projects_for_user] \ + if projects_for_user else ret + + def get_projects_for_api_user(self, region, token): + if not token: + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + endpoint = self.get_region_url_nover(region, "keystone") + headers = { + 'X-Auth-Project-Id': self.admin_project, + 'X-Auth-Token': token['id'] + } + # get the list of projects accessible by the admin user + req_url = endpoint + '/v3/projects' + response = self.get_url(req_url, headers) + if not response or 'projects' not in response: + return None + response = [p['name'] for p in response['projects']] + return response + + def get_for_region(self, region, token): + endpoint = self.get_region_url_nover(region, "keystone") + req_url = endpoint + "/v2.0/tenants" + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not isinstance(response, dict): + self.log.error('invalid response to /tenants request: not dict') + return [] + tenants_list = response.get("tenants", []) + if not isinstance(tenants_list, list): + self.log.error('invalid response to /tenants request: ' + 'tenants value is n ot a list') + return [] + response = [t for t in tenants_list if t.get("name", "") != "services"] + return response diff --git a/app/discover/fetchers/api/api_fetch_regions.py b/app/discover/fetchers/api/api_fetch_regions.py new file mode 100644 index 0000000..dcc558f --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_regions.py @@ -0,0 +1,51 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess + + +class ApiFetchRegions(ApiAccess): + def __init__(self): + super(ApiFetchRegions, self).__init__() + self.endpoint = ApiAccess.base_url + + def get(self, project_id): + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + # the returned authentication response contains the list of end points + # and regions + service_catalog = ApiAccess.auth_response.get('access', {}).get('serviceCatalog') + if not service_catalog: + return [] + env = self.get_env() + ret = [] + NULL_REGION = "No-Region" + for service in service_catalog: + for e in service["endpoints"]: + if "region" in e: + region_name = e.pop("region") + region_name = region_name if region_name else NULL_REGION + else: + region_name = NULL_REGION + if region_name in self.regions.keys(): + region = self.regions[region_name] + else: + region = { + "id": region_name, + "name": region_name, + "endpoints": {} + } + ApiAccess.regions[region_name] = region + region["parent_type"] = "regions_folder" + region["parent_id"] = env + "-regions" + e["service_type"] = service["type"] + region["endpoints"][service["name"]] = e + ret.extend(list(ApiAccess.regions.values())) + return ret diff --git a/app/discover/fetchers/cli/__init__.py b/app/discover/fetchers/cli/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/cli/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py new file mode 100644 index 0000000..1db84ea --- /dev/null +++ b/app/discover/fetchers/cli/cli_access.py @@ -0,0 +1,206 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re +import time + +from discover.fetcher import Fetcher +from utils.binary_converter import BinaryConverter +from utils.logging.console_logger import ConsoleLogger +from utils.ssh_conn import SshConn + + +class CliAccess(BinaryConverter, Fetcher): + connections = {} + ssh_cmd = "ssh -o StrictHostKeyChecking=no " + call_count_per_con = {} + max_call_count_per_con = 100 + cache_lifetime = 60 # no. of seconds to cache results + cached_commands = {} + + def __init__(self): + super().__init__() + self.log = ConsoleLogger() + + @staticmethod + def is_gateway_host(ssh_to_host): + ssh_conn = SshConn(ssh_to_host) + return ssh_conn.is_gateway_host(ssh_to_host) + + def run_on_gateway(self, cmd, ssh_to_host="", enable_cache=True, + use_sudo=True): + self.run(cmd, ssh_to_host=ssh_to_host, enable_cache=enable_cache, + on_gateway=True, use_sudo=use_sudo) + + def run(self, cmd, ssh_to_host="", enable_cache=True, on_gateway=False, + ssh=None, use_sudo=True): + ssh_conn = ssh if ssh else SshConn(ssh_to_host) + if use_sudo and not cmd.strip().startswith("sudo "): + cmd = "sudo " + cmd + if not on_gateway and ssh_to_host \ + and not ssh_conn.is_gateway_host(ssh_to_host): + cmd = self.ssh_cmd + ssh_to_host + " " + cmd + curr_time = time.time() + cmd_path = ssh_to_host + ',' + cmd + if enable_cache and cmd_path in self.cached_commands: + # try to re-use output from last call + cached = self.cached_commands[cmd_path] + if cached["timestamp"] + self.cache_lifetime < curr_time: + # result expired + self.cached_commands.pop(cmd_path, None) + else: + # result is good to use - skip the SSH call + self.log.info('CliAccess: ****** using cached result, ' + + 'host: ' + ssh_to_host + ', cmd: %s ******', cmd) + return cached["result"] + + self.log.info('CliAccess: host: %s, cmd: %s', ssh_to_host, cmd) + ret = ssh_conn.exec(cmd) + self.cached_commands[cmd_path] = {"timestamp": curr_time, "result": ret} + return ret + + def run_fetch_lines(self, cmd, ssh_to_host="", enable_cache=True): + out = self.run(cmd, ssh_to_host, enable_cache) + if not out: + return [] + # first try to split lines by whitespace + ret = out.splitlines() + # if split by whitespace did not work, try splitting by "\\n" + if len(ret) == 1: + ret = [l for l in out.split("\\n") if l != ""] + return ret + + # parse command output columns separated by whitespace + # since headers can contain whitespace themselves, + # it is the caller's responsibility to provide the headers + def parse_cmd_result_with_whitespace(self, lines, headers, remove_first): + if remove_first: + # remove headers line + del lines[:1] + results = [self.parse_line_with_ws(line, headers) + for line in lines] + return results + + # parse command output with "|" column separators and "-" row separators + def parse_cmd_result_with_separators(self, lines): + headers = self.parse_headers_line_with_separators(lines[1]) + # remove line with headers and formatting lines above it and below it + del lines[:3] + # remove formatting line in the end + lines.pop() + results = [self.parse_content_line_with_separators(line, headers) + for line in lines] + return results + + # parse a line with columns separated by whitespace + def parse_line_with_ws(self, line, headers): + s = line if isinstance(line, str) else self.binary2str(line) + parts = [word.strip() for word in s.split() if word.strip()] + ret = {} + for i, p in enumerate(parts): + header = headers[i] + ret[header] = p + return ret + + # parse a line with "|" column separators + def parse_line_with_separators(self, line): + s = self.binary2str(line) + parts = [word.strip() for word in s.split("|") if word.strip()] + # remove the ID field + del parts[:1] + return parts + + def parse_headers_line_with_separators(self, line): + return self.parse_line_with_separators(line) + + def parse_content_line_with_separators(self, line, headers): + content_parts = self.parse_line_with_separators(line) + content = {} + for i in range(0, len(content_parts)): + content[headers[i]] = content_parts[i] + return content + + def merge_ws_spillover_lines(self, lines): + # with WS-separated output, extra output sometimes spills to next line + # detect that and add to the end of the previous line for our procesing + pending_line = None + fixed_lines = [] + # remove headers line + for l in lines: + if l[0] == '\t': + # this is a spill-over line + if pending_line: + # add this line to the end of the previous line + pending_line = pending_line.strip() + "," + l.strip() + else: + # add the previous pending line to the fixed lines list + if pending_line: + fixed_lines.append(pending_line) + # make current line the pending line + pending_line = l + if pending_line: + fixed_lines.append(pending_line) + return fixed_lines + + """ + given output lines from CLI command like 'ip -d link show', + find lines belonging to section describing a specific interface + parameters: + - lines: list of strings, output of command + - header_regexp: regexp marking the start of the section + - end_regexp: regexp marking the end of the section + """ + def get_section_lines(self, lines, header_regexp, end_regexp): + if not lines: + return [] + header_re = re.compile(header_regexp) + start_pos = None + # find start_pos of section + line_count = len(lines) + for line_num in range(0, line_count-1): + matches = header_re.match(lines[line_num]) + if matches: + start_pos = line_num + break + if not start_pos: + return [] + # find end of section + end_pos = line_count + end_re = re.compile(end_regexp) + for line_num in range(start_pos+1, end_pos-1): + matches = end_re.match(lines[line_num]) + if matches: + end_pos = line_num + break + return lines[start_pos:end_pos] + + def get_object_data(self, o, lines, regexps): + """ + find object data in output lines from CLI command + parameters: + - o: object (dict), to which we'll add attributes with the data found + - lines: list of strings + - regexps: dict, keys are attribute names, values are regexp to match + for finding the value of the attribute + """ + for line in lines: + self.find_matching_regexps(o, line, regexps) + for regexp_tuple in regexps: + name = regexp_tuple['name'] + if 'name' not in o and 'default' in regexp_tuple: + o[name] = regexp_tuple['default'] + + def find_matching_regexps(self, o, line, regexps): + for regexp_tuple in regexps: + name = regexp_tuple['name'] + regex = regexp_tuple['re'] + regex = re.compile(regex) + matches = regex.search(line) + if matches: + o[name] = matches.group(1) diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py new file mode 100644 index 0000000..3516e25 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py @@ -0,0 +1,122 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchHostPnics(CliAccess): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.ethtool_attr = re.compile('^\s+([^:]+):\s(.*)$') + self.regexps = [ + {'name': 'mac_address', 're': '^.*\sHWaddr\s(\S+)(\s.*)?$'}, + {'name': 'mac_address', 're': '^.*\sether\s(\S+)(\s.*)?$'}, + {'name': 'IP Address', 're': '^\s*inet addr:?(\S+)\s.*$'}, + {'name': 'IP Address', 're': '^\s*inet ([0-9.]+)\s.*$'}, + {'name': 'IPv6 Address', 're': '^\s*inet6 addr:\s*(\S+)(\s.*)?$'}, + {'name': 'IPv6 Address', 're': '^\s*inet6 \s*(\S+)(\s.*)?$'} + ] + + def get(self, id): + host_id = id[:id.rindex("-")] + cmd = 'ls -l /sys/class/net | grep ^l | grep -v "/virtual/"' + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error("CliFetchHostPnics: host not found: " + host_id) + return [] + if "host_type" not in host: + self.log.error("host does not have host_type: " + host_id + + ", host: " + str(host)) + return [] + host_types = host["host_type"] + if "Network" not in host_types and "Compute" not in host_types: + return [] + interface_lines = self.run_fetch_lines(cmd, host_id) + interfaces = [] + for line in interface_lines: + interface_name = line[line.rindex('/')+1:] + interface_name = interface_name.strip() + # run ifconfig with specific interface name, + # since running it with no name yields a list without inactive pNICs + interface = self.find_interface_details(host_id, interface_name) + if interface: + interfaces.append(interface) + return interfaces + + def find_interface_details(self, host_id, interface_name): + lines = self.run_fetch_lines("ifconfig " + interface_name, host_id) + interface = None + status_up = None + for line in [l for l in lines if l != '']: + tokens = None + if interface is None: + tokens = line.split() + name = tokens[0].strip('- :') + name = name.strip() + if name == interface_name: + line_remainder = line.strip('-')[len(interface_name)+2:] + line_remainder = line_remainder.strip(' :') + id = interface_name + interface = { + "host": host_id, + "name": id, + "local_name": interface_name, + "lines": [] + } + self.handle_line(interface, line_remainder) + if '<UP,' in line: + status_up = True + if status_up is None: + if tokens is None: + tokens = line.split() + if 'BROADCAST' in tokens: + status_up = 'UP' in tokens + if interface: + self.handle_line(interface, line) + self.set_interface_data(interface) + interface['state'] = 'UP' if status_up else 'DOWN' + if 'id' not in interface: + interface['id'] = interface_name + '-unknown_mac' + return interface + + def handle_line(self, interface, line): + self.find_matching_regexps(interface, line, self.regexps) + if 'mac_address' in interface: + interface["id"] = interface["name"] + "-" + interface["mac_address"] + interface["lines"].append(line.strip()) + + def set_interface_data(self, interface): + if not interface: + return + interface["data"] = "\n".join(interface["lines"]) + interface.pop("lines", None) + ethtool_ifname = interface["local_name"] + if "@" in interface["local_name"]: + pos = interface["local_name"].index("@") + ethtool_ifname = ethtool_ifname[pos + 1:] + cmd = "ethtool " + ethtool_ifname + lines = self.run_fetch_lines(cmd, interface["host"]) + attr = None + for line in lines[1:]: + matches = self.ethtool_attr.match(line) + if matches: + # add this attribute to the interface + attr = matches.group(1) + value = matches.group(2) + interface[attr] = value.strip() + else: + # add more values to the current attribute as an array + if isinstance(interface[attr], str): + interface[attr] = [interface[attr], line.strip()] + else: + interface[attr].append(line.strip()) diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics_vpp.py b/app/discover/fetchers/cli/cli_fetch_host_pnics_vpp.py new file mode 100644 index 0000000..69b6413 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_host_pnics_vpp.py @@ -0,0 +1,44 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetcher import Fetcher +from utils.inventory_mgr import InventoryMgr + +NAME_RE = '^[a-zA-Z]*GigabitEthernet' + +class CliFetchHostPnicsVpp(Fetcher): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.name_re = re.compile(NAME_RE) + + def get(self, id): + host_id = id[:id.rindex("-")] + host_id = id[:host_id.rindex("-")] + vedges = self.inv.find_items({ + "environment": self.get_env(), + "type": "vedge", + "host": host_id + }) + ret = [] + for vedge in vedges: + pnic_ports = vedge['ports'] + for pnic_name in pnic_ports: + if not self.name_re.search(pnic_name): + continue + pnic = pnic_ports[pnic_name] + pnic['host'] = host_id + pnic['id'] = host_id + "-pnic-" + pnic_name + pnic['type'] = 'pnic' + pnic['object_name'] = pnic_name + pnic['Link detected'] = 'yes' if pnic['state'] == 'up' else 'no' + ret.append(pnic) + return ret diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservice.py b/app/discover/fetchers/cli/cli_fetch_host_vservice.py new file mode 100644 index 0000000..9f8173f --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_host_vservice.py @@ -0,0 +1,80 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.cli.cli_access import CliAccess +from discover.fetchers.db.db_access import DbAccess +from discover.network_agents_list import NetworkAgentsList +from utils.inventory_mgr import InventoryMgr + + +class CliFetchHostVservice(CliAccess, DbAccess): + def __init__(self): + super(CliFetchHostVservice, self).__init__() + # match only DHCP agent and router (L3 agent) + self.type_re = re.compile("^q(dhcp|router)-") + self.inv = InventoryMgr() + self.agents_list = NetworkAgentsList() + + def get_vservice(self, host_id, name_space): + result = {"local_service_id": name_space} + self.set_details(host_id, result) + return result + + def set_details(self, host_id, r): + # keep the index without prefix + id_full = r["local_service_id"].strip() + prefix = id_full[1:id_full.index('-')] + id_clean = id_full[id_full.index('-') + 1:] + r["service_type"] = prefix + name = self.get_router_name(r, id_clean) if prefix == "router" \ + else self.get_network_name(id_clean) + r["name"] = prefix + "-" + name + r["host"] = host_id + r["id"] = host_id + "-" + id_full + self.set_agent_type(r) + + def get_network_name(self, id): + query = """ + SELECT name + FROM {}.networks + WHERE id = %s + """.format(self.neutron_db) + results = self.get_objects_list_for_id(query, "router", id) + if not list(results): + return id + for db_row in results: + return db_row["name"] + + def get_router_name(self, r, id): + query = """ + SELECT * + FROM {}.routers + WHERE id = %s + """.format(self.neutron_db) + results = self.get_objects_list_for_id(query, "router", id.strip()) + for db_row in results: + r.update(db_row) + return r["name"] + + # dynamically create sub-folder for vService by type + def set_agent_type(self, o): + o["master_parent_id"] = o["host"] + "-vservices" + o["master_parent_type"] = "vservices_folder" + atype = o["service_type"] + agent = self.agents_list.get_type(atype) + try: + o["parent_id"] = o["master_parent_id"] + "-" + agent["type"] + "s" + o["parent_type"] = "vservice_" + agent["type"] + "s_folder" + o["parent_text"] = agent["folder_text"] + except KeyError: + o["parent_id"] = o["master_parent_id"] + "-" + "miscellenaous" + o["parent_type"] = "vservice_miscellenaous_folder" + o["parent_text"] = "Misc. services" diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservices.py b/app/discover/fetchers/cli/cli_fetch_host_vservices.py new file mode 100644 index 0000000..9b62dcb --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_host_vservices.py @@ -0,0 +1,27 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_fetch_host_vservice import CliFetchHostVservice + + +class CliFetchHostVservices(CliFetchHostVservice): + def __init__(self): + super(CliFetchHostVservices, self).__init__() + + def get(self, host_id): + host = self.inv.get_single(self.get_env(), "host", host_id) + if "Network" not in host["host_type"]: + return [] + services_ids = [l[:l.index(' ')] if ' ' in l else l + for l in self.run_fetch_lines("ip netns", host_id)] + results = [{"local_service_id": s} for s in services_ids if self.type_re.match(s)] + for r in results: + self.set_details(host_id, r) + return results + diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics.py new file mode 100644 index 0000000..22ac573 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics.py @@ -0,0 +1,22 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_fetch_instance_vnics_base import CliFetchInstanceVnicsBase + + +class CliFetchInstanceVnics(CliFetchInstanceVnicsBase): + def __init__(self): + super().__init__() + + def set_vnic_properties(self, v, instance): + super().set_vnic_properties(v, instance) + v["source_bridge"] = v["source"]["@bridge"] + + def get_vnic_name(self, v, instance): + return v["target"]["@dev"] diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py new file mode 100644 index 0000000..4de1840 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py @@ -0,0 +1,68 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import xmltodict + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchInstanceVnicsBase(CliAccess): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def get(self, id): + instance_uuid = id[:id.rindex('-')] + instance = self.inv.get_by_id(self.get_env(), instance_uuid) + if not instance: + return [] + host = self.inv.get_by_id(self.get_env(), instance["host"]) + if not host or "Compute" not in host["host_type"]: + return [] + lines = self.run_fetch_lines("virsh list", instance["host"]) + del lines[:2] # remove header + virsh_ids = [l.split()[0] for l in lines if l > ""] + results = [] + # Note: there are 2 ids here of instances with local names, which are + # not connected to the data we have thus far for the instance + # therefore, we will decide whether the instance is the correct one + # based on comparison of the uuid in the dumpxml output + for id in virsh_ids: + results.extend(self.get_vnics_from_dumpxml(id, instance)) + return results + + def get_vnics_from_dumpxml(self, id, instance): + xml_string = self.run("virsh dumpxml " + id, instance["host"]) + if not xml_string.strip(): + return [] + response = xmltodict.parse(xml_string) + if instance["uuid"] != response["domain"]["uuid"]: + # this is the wrong instance - skip it + return [] + try: + vnics = response["domain"]["devices"]["interface"] + except KeyError: + return [] + if isinstance(vnics, dict): + vnics = [vnics] + for v in vnics: + self.set_vnic_properties(v, instance) + return vnics + + def set_vnic_properties(self, v, instance): + v["name"] = self.get_vnic_name(v, instance) + v["id"] = v["name"] + v["vnic_type"] = "instance_vnic" + v["host"] = instance["host"] + v["instance_id"] = instance["id"] + v["instance_db_id"] = instance["_id"] + v["mac_address"] = v["mac"]["@address"] + instance["mac_address"] = v["mac_address"] + self.inv.set(instance) diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py new file mode 100644 index 0000000..58facd2 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py @@ -0,0 +1,18 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_fetch_instance_vnics_base import CliFetchInstanceVnicsBase + + +class CliFetchInstanceVnicsVpp(CliFetchInstanceVnicsBase): + def __init__(self): + super().__init__() + + def get_vnic_name(self, v, instance): + return instance["name"] + "-" + v["@type"] + "-" + v["mac"]["@address"] diff --git a/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py b/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py new file mode 100644 index 0000000..1e65a14 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py @@ -0,0 +1,86 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_access import CliAccess +from discover.fetchers.db.db_access import DbAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchOtepsLxb(CliAccess, DbAccess): + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def get(self, parent_id): + vconnector = self.inv.get_by_id(self.get_env(), parent_id) + if not vconnector: + return [] + configurations = vconnector['configurations'] + tunneling_ip = configurations['tunneling_ip'] + tunnel_types_used = configurations['tunnel_types'] + if not tunnel_types_used: + return [] + tunnel_type = tunnel_types_used[0] + if not tunnel_type: + return [] + # check only interfaces with name matching tunnel type + ret = [i for i in vconnector['interfaces'].values() + if i['name'].startswith(tunnel_type + '-')] + for otep in ret: + otep['ip_address'] = tunneling_ip + otep['host'] = vconnector['host'] + self.get_otep_ports(otep) + otep['id'] = otep['host'] + '-otep-' + otep['name'] + otep['name'] = otep['id'] + otep['vconnector'] = vconnector['name'] + otep['overlay_type'] = tunnel_type + self.get_udp_port(otep) + return ret + + """ + fetch OTEP data from CLI command 'ip -d link show' + """ + def get_otep_ports(self, otep): + cmd = 'ip -d link show' + lines = self.run_fetch_lines(cmd, otep['host']) + header_format = '[0-9]+: ' + otep['name'] + ':' + interface_lines = self.get_section_lines(lines, header_format, '\S') + otep['data'] = '\n'.join(interface_lines) + regexps = [ + {'name': 'state', 're': ',UP,', 'default': 'DOWN'}, + {'name': 'mac_address', 're': '.*\slink/ether\s(\S+)\s'}, + {'name': 'mtu', 're': '.*\smtu\s(\S+)\s'}, + ] + self.get_object_data(otep, interface_lines, regexps) + cmd = 'bridge fdb show' + dst_line_format = ' dev ' + otep['name'] + ' dst ' + lines = self.run_fetch_lines(cmd, otep['host']) + lines = [l for l in lines if dst_line_format in l] + if lines: + l = lines[0] + otep['bridge dst'] = l[l.index(' dst ')+5:] + return otep + + def get_udp_port(self, otep): + table_name = "neutron.ml2_" + otep['overlay_type'] + "_endpoints" + results = None + try: + results = self.get_objects_list_for_id( + """ + SELECT udp_port + FROM {} + WHERE host = %s + """.format(table_name), + "vedge", otep['host']) + except Exception as e: + self.log.error('failed to fetch UDP port for OTEP: ' + str(e)) + otep['udp_port'] = 0 + for result in results: + otep['udp_port'] = result['udp_port'] diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors.py b/app/discover/fetchers/cli/cli_fetch_vconnectors.py new file mode 100644 index 0000000..78b767a --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors.py @@ -0,0 +1,40 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from abc import abstractmethod, ABCMeta + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr +from utils.singleton import Singleton + + +class ABCSingleton(ABCMeta, Singleton): + pass + + +class CliFetchVconnectors(CliAccess, metaclass=ABCSingleton): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + @abstractmethod + def get_vconnectors(self, host): + raise NotImplementedError("Subclass must override get_vconnectors()") + + def get(self, id): + host_id = id[:id.rindex('-')] + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error("CliFetchVconnectors: host not found: " + host_id) + return [] + if "host_type" not in host: + self.log.error("host does not have host_type: " + host_id + \ + ", host: " + str(host)) + return [] + return self.get_vconnectors(host) diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py new file mode 100644 index 0000000..648dc63 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py @@ -0,0 +1,35 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +from discover.fetchers.cli.cli_fetch_vconnectors_ovs import CliFetchVconnectorsOvs +from discover.fetchers.db.db_access import DbAccess + + +class CliFetchVconnectorsLxb(CliFetchVconnectorsOvs, DbAccess): + + def __init__(self): + super().__init__() + + def get(self, id): + ret = super().get(id) + for doc in ret: + query = """ + SELECT configurations + FROM {}.agents + WHERE agent_type="Linux bridge agent" AND host = %s + """.format(self.neutron_db) + host = doc['host'] + matches = self.get_objects_list_for_id(query, '', host) + if not matches: + raise ValueError('No Linux bridge agent in DB for host: {}'.format(host)) + agent = matches[0] + doc['configurations'] = json.loads(agent['configurations']) + return ret diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py new file mode 100644 index 0000000..ff37569 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py @@ -0,0 +1,56 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.cli.cli_fetch_vconnectors import CliFetchVconnectors + + +class CliFetchVconnectorsOvs(CliFetchVconnectors): + def __init__(self): + super().__init__() + + def get_vconnectors(self, host): + host_id = host['id'] + lines = self.run_fetch_lines("brctl show", host_id) + headers = ["bridge_name", "bridge_id", "stp_enabled", "interfaces"] + headers_count = len(headers) + # since we hard-coded the headers list, remove the headers line + del lines[:1] + + # intefaces can spill to next line - need to detect that and add + # them to the end of the previous line for our procesing + fixed_lines = self.merge_ws_spillover_lines(lines) + + results = self.parse_cmd_result_with_whitespace(fixed_lines, headers, False) + ret = [] + for doc in results: + doc["name"] = doc.pop("bridge_name") + doc["id"] = doc["name"] + "-" + doc.pop("bridge_id") + doc["host"] = host_id + doc["connector_type"] = "bridge" + if "interfaces" in doc: + interfaces = {} + interface_names = doc["interfaces"].split(",") + for interface_name in interface_names: + # find MAC address for this interface from ports list + port_id_prefix = interface_name[3:] + port = self.inv.find_items({ + "environment": self.get_env(), + "type": "port", + "binding:host_id": host_id, + "id": {"$regex": r"^" + re.escape(port_id_prefix)} + }, get_single=True) + mac_address = '' if not port else port['mac_address'] + interface = {'name': interface_name, 'mac_address': mac_address} + interfaces[interface_name] = interface + doc["interfaces"] = interfaces + doc['interfaces_names'] = list(interfaces.keys()) + ret.append(doc) + return ret diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py new file mode 100644 index 0000000..479e1db --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py @@ -0,0 +1,64 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_fetch_vconnectors import CliFetchVconnectors + + +class CliFetchVconnectorsVpp(CliFetchVconnectors): + def __init__(self): + super().__init__() + + def get_vconnectors(self, host): + lines = self.run_fetch_lines("vppctl show mode", host['id']) + vconnectors = {} + for l in lines: + if not l.startswith('l2 bridge'): + continue + line_parts = l.split(' ') + name = line_parts[2] + bd_id = line_parts[4] + if bd_id in vconnectors: + vconnector = vconnectors[bd_id] + else: + vconnector = { + 'host': host['id'], + 'id': host['id'] + '-vconnector-' + bd_id, + 'bd_id': bd_id, + 'name': "bridge-domain-" + bd_id, + 'interfaces': {}, + 'interfaces_names': [] + } + vconnectors[bd_id] = vconnector + interface = self.get_interface_details(host, name) + if interface: + vconnector['interfaces'][name] = interface + vconnector['interfaces_names'].append(name) + return list(vconnectors.values()) + + def get_interface_details(self, host, name): + # find vconnector interfaces + cmd = "vppctl show hardware-int " + name + interface_lines = self.run_fetch_lines(cmd, host['id']) + # remove header line + interface_lines.pop(0) + interface = None + for l in interface_lines: + if not l.strip(): + continue # ignore empty lines + if not l.startswith(' '): + details = l.split() + interface = { + "name": details[0], + "hardware": details[3], + "state": details[2], + "id": details[1], + } + elif l.startswith(' Ethernet address '): + interface['mac_address'] = l[l.rindex(' ') + 1:] + return interface diff --git a/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py b/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py new file mode 100644 index 0000000..f9c622d --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py @@ -0,0 +1,58 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# Copyright 2016 cisco Corporation +#oslo related message handling + +from oslo_serialization import jsonutils +from oslo_utils import uuidutils +import yaml + +from neutronclient.tests.functional import base + + +class TestCLIFormatter(base.ClientTestBase): + +## old stuff ..not related to vpp..disregard + def setUp(self): + super(TestCLIFormatter, self).setUp() + self.net_name = 'net-%s' % uuidutils.generate_uuid() + self.addCleanup(self.neutron, 'net-delete %s' % self.net_name) + + def _create_net(self, fmt, col_attrs): + params = ['-c %s' % attr for attr in col_attrs] + params.append('-f %s' % fmt) + params.append(self.net_name) + param_string = ' '.join(params) + return self.neutron('net-create', params=param_string) + + def test_net_create_with_json_formatter(self): + result = self._create_net('json', ['name', 'admin_state_up']) + self.assertDictEqual({'name': self.net_name, + 'admin_state_up': True}, + jsonutils.loads(result)) + + def test_net_create_with_yaml_formatter(self): + result = self._create_net('yaml', ['name', 'admin_state_up']) + self.assertDictEqual({'name': self.net_name, + 'admin_state_up': True}, + yaml.load(result)) + + def test_net_create_with_value_formatter(self): + # NOTE(amotoki): In 'value' formatter, there is no guarantee + # in the order of attribute, so we use one attribute in this test. + result = self._create_net('value', ['name']) + self.assertEqual(self.net_name, result.strip()) + + def test_net_create_with_shell_formatter(self): + result = self._create_net('shell', ['name', 'admin_state_up']) + result_lines = set(result.strip().split('\n')) + self.assertSetEqual(set(['name="%s"' % self.net_name, + 'admin_state_up="True"']), +result_lines) diff --git a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py new file mode 100644 index 0000000..44ac8d6 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py @@ -0,0 +1,140 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchVserviceVnics(CliAccess): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.if_header = re.compile('^[-]?(\S+)\s+(.*)$') + self.regexps = [ + {'name': 'mac_address', 're': '^.*\sHWaddr\s(\S+)(\s.*)?$'}, + {'name': 'mac_address', 're': '^.*\sether\s(\S+)(\s.*)?$'}, + {'name': 'netmask', 're': '^.*\sMask:\s?([0-9.]+)(\s.*)?$'}, + {'name': 'netmask', 're': '^.*\snetmask\s([0-9.]+)(\s.*)?$'}, + {'name': 'IP Address', 're': '^\s*inet addr:(\S+)\s.*$'}, + {'name': 'IP Address', 're': '^\s*inet ([0-9.]+)\s.*$'}, + {'name': 'IPv6 Address', + 're': '^\s*inet6 addr: ?\s*([0-9a-f:/]+)(\s.*)?$'}, + {'name': 'IPv6 Address', + 're': '^\s*inet6 \s*([0-9a-f:/]+)(\s.*)?$'} + ] + + def get(self, host_id): + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error("host not found: " + host_id) + return [] + if "host_type" not in host: + self.log.error("host does not have host_type: " + host_id + + ", host: " + str(host)) + return [] + if "Network" not in host["host_type"]: + return [] + lines = self.run_fetch_lines("ip netns", host_id) + ret = [] + for l in [l for l in lines + if l.startswith("qdhcp") or l.startswith("qrouter")]: + service = l.strip() + service = service if ' ' not in service \ + else service[:service.index(' ')] + ret.extend(self.handle_service(host_id, service)) + return ret + + def handle_service(self, host, service, enable_cache=True): + cmd = "ip netns exec " + service + " ifconfig" + lines = self.run_fetch_lines(cmd, host, enable_cache) + interfaces = [] + current = None + for line in lines: + matches = self.if_header.match(line) + if matches: + if current: + self.set_interface_data(current) + name = matches.group(1).strip(":") + # ignore 'lo' interface + if name == 'lo': + current = None + else: + line_remainder = matches.group(2) + vservice_id = host + "-" + service + current = { + "id": host + "-" + name, + "type": "vnic", + "vnic_type": "vservice_vnic", + "host": host, + "name": name, + "master_parent_type": "vservice", + "master_parent_id": vservice_id, + "parent_type": "vnics_folder", + "parent_id": vservice_id + "-vnics", + "parent_text": "vNICs", + "lines": [] + } + interfaces.append(current) + self.handle_line(current, line_remainder) + else: + if current: + self.handle_line(current, line) + if current: + self.set_interface_data(current) + return interfaces + + def handle_line(self, interface, line): + self.find_matching_regexps(interface, line, self.regexps) + interface["lines"].append(line.strip()) + + def set_interface_data(self, interface): + if not interface or 'IP Address' not in interface or 'netmask' not in interface: + return + + interface["data"] = "\n".join(interface.pop("lines", None)) + interface["cidr"] = self.get_cidr_for_vnic(interface) + network = self.inv.get_by_field(self.get_env(), "network", "cidrs", + interface["cidr"], get_single=True) + if not network: + return + interface["network"] = network["id"] + # set network for the vservice, to check network on clique creation + vservice = self.inv.get_by_id(self.get_env(), + interface["master_parent_id"]) + network_id = network["id"] + if "network" not in vservice: + vservice["network"] = list() + if network_id not in vservice["network"]: + vservice["network"].append(network_id) + self.inv.set(vservice) + + # find CIDR string by IP address and netmask + def get_cidr_for_vnic(self, vnic): + if "IP Address" not in vnic: + vnic["IP Address"] = "No IP Address" + return "No IP Address" + ipaddr = vnic["IP Address"].split('.') + netmask = vnic["netmask"].split('.') + + # calculate network start + net_start = [] + for pos in range(0, 4): + net_start.append(str(int(ipaddr[pos]) & int(netmask[pos]))) + + cidr_string = '.'.join(net_start) + '/' + cidr_string = cidr_string + self.get_net_size(netmask) + return cidr_string + + def get_net_size(self, netmask): + binary_str = '' + for octet in netmask: + binary_str += bin(int(octet))[2:].zfill(8) + return str(len(binary_str.rstrip('0'))) diff --git a/app/discover/fetchers/db/__init__.py b/app/discover/fetchers/db/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/db/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/db/db_access.py b/app/discover/fetchers/db/db_access.py new file mode 100644 index 0000000..00bd776 --- /dev/null +++ b/app/discover/fetchers/db/db_access.py @@ -0,0 +1,142 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import mysql.connector + +from discover.configuration import Configuration +from discover.fetcher import Fetcher +from utils.string_utils import jsonify + + +class DbAccess(Fetcher): + conn = None + query_count_per_con = 0 + + # connection timeout set to 30 seconds, + # due to problems over long connections + TIMEOUT = 30 + + def __init__(self): + super().__init__() + self.config = Configuration() + self.conf = self.config.get("mysql") + self.connect_to_db() + cursor = DbAccess.conn.cursor(dictionary=True) + try: + # check if DB schema 'neutron' exists + cursor.execute("SELECT COUNT(*) FROM neutron.agents") + for row in cursor: + pass + self.neutron_db = "neutron" + except (AttributeError, mysql.connector.errors.ProgrammingError): + self.neutron_db = "ml2_neutron" + + def db_connect(self, _host, _port, _user, _password, _database): + if DbAccess.conn: + return + try: + connector = mysql.connector + DbAccess.conn = connector.connect(host=_host, port=_port, + connection_timeout=self.TIMEOUT, + user=_user, + password=_password, + database=_database, + raise_on_warnings=True) + DbAccess.conn.ping(True) # auto-reconnect if necessary + except: + self.log.critical("failed to connect to MySQL DB") + return + DbAccess.query_count_per_con = 0 + + def connect_to_db(self, force=False): + if DbAccess.conn: + if not force: + return + self.log.info("DbAccess: ****** forcing reconnect, " + + "query count: %s ******", + DbAccess.query_count_per_con) + DbAccess.conn = None + self.conf = self.config.get("mysql") + cnf = self.conf + cnf['schema'] = cnf['schema'] if 'schema' in cnf else 'nova' + self.db_connect(cnf["host"], cnf["port"], + cnf["user"], cnf["password"], + cnf["schema"]) + + def get_objects_list_for_id(self, query, object_type, id): + self.connect_to_db(DbAccess.query_count_per_con >= 25) + DbAccess.query_count_per_con += 1 + self.log.debug("query count: %s, running query:\n%s\n", + str(DbAccess.query_count_per_con), query) + + cursor = DbAccess.conn.cursor(dictionary=True) + try: + if id: + cursor.execute(query, [str(id)]) + else: + cursor.execute(query) + except (AttributeError, mysql.connector.errors.OperationalError) as e: + self.log.error(e) + self.connect_to_db(True) + # try again to run the query + cursor = DbAccess.conn.cursor(dictionary=True) + if id: + cursor.execute(query, [str(id)]) + else: + cursor.execute(query) + + rows = [] + for row in cursor: + rows.append(row) + return rows + + def get_objects_list(self, query, object_type): + return self.get_objects_list_for_id(query, object_type, None) + + def get_objects(self, qry, type, id): + return jsonify(self.get_objects_list(qry, type)) + + def get(self, id): + # return list of available fetch types + ret = { + "description": "List of available fetch calls for this interface", + "types": { + "regions": "Regions of this environment", + "projects": "Projects (tenants) of this environment", + "availability_zones": "Availability zones", + "aggregates": "Host aggregates", + "aggregate_hosts": "Hosts in aggregate X (parameter: id)", + "az_hosts": "Host in availability_zone X (parameter: id)" + } + } + return jsonify(ret) + + def exec(self, query, table, field, values): + try: + cursor = DbAccess.conn.cursor(dictionary=True) + cursor.execute(query, [table, field, values]) + except (AttributeError, mysql.connector.errors.OperationalError) as e: + self.log.error(e) + self.connect_to_db(True) + # try again to run the query + cursor = DbAccess.conn.cursor(dictionary=True) + cursor.execute(query, [table, field, values]) + + rows = [] + for row in cursor: + rows.append(row) + return rows + + def set(self, table, field, values): + query = """INSERT INTO %s %s VALUES %s""" + return self.exec(query, table, field, values) + + def delete(self, table, field, values): + query = """DELETE FROM %s WHERE %s=%s""" + return self.exec(query, table, field, values) diff --git a/app/discover/fetchers/db/db_fetch_aggregate_hosts.py b/app/discover/fetchers/db/db_fetch_aggregate_hosts.py new file mode 100644 index 0000000..59ba5d0 --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_aggregate_hosts.py @@ -0,0 +1,36 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import bson + +from discover.fetchers.db.db_access import DbAccess +from utils.inventory_mgr import InventoryMgr + + +class DbFetchAggregateHosts(DbAccess): + def get(self, id): + query = """ + SELECT CONCAT('aggregate-', a.name, '-', host) AS id, host AS name + FROM nova.aggregate_hosts ah + JOIN nova.aggregates a ON a.id = ah.aggregate_id + WHERE ah.deleted = 0 AND aggregate_id = %s + """ + hosts = self.get_objects_list_for_id(query, "host", id) + if hosts: + inv = InventoryMgr() + for host_rec in hosts: + host_id = host_rec['name'] + host = inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error('unable to find host {} ' + 'from aggregate {} in inventory' + .format(host_id, id)) + continue + host_rec['ref_id'] = bson.ObjectId(host['_id']) + return hosts diff --git a/app/discover/fetchers/db/db_fetch_aggregates.py b/app/discover/fetchers/db/db_fetch_aggregates.py new file mode 100644 index 0000000..da0720b --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_aggregates.py @@ -0,0 +1,21 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.db.db_access import DbAccess + + +class DbFetchAggregates(DbAccess): + def get(self, id): + return self.get_objects_list( + """ + SELECT id, name + FROM nova.aggregates + WHERE deleted = 0 + """, + "host aggregate") diff --git a/app/discover/fetchers/db/db_fetch_availability_zones.py b/app/discover/fetchers/db/db_fetch_availability_zones.py new file mode 100644 index 0000000..763d777 --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_availability_zones.py @@ -0,0 +1,22 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.db.db_access import DbAccess + +class DbFetchAvailabilityZones(DbAccess): + + def get(self, id): + query = """ + SELECT DISTINCT availability_zone, + availability_zone AS id, COUNT(DISTINCT host) AS descendants + FROM nova.instances + WHERE availability_zone IS NOT NULL + GROUP BY availability_zone + """ + return self.get_objects_list(query, "availability zone") diff --git a/app/discover/fetchers/db/db_fetch_az_network_hosts.py b/app/discover/fetchers/db/db_fetch_az_network_hosts.py new file mode 100644 index 0000000..09043ea --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_az_network_hosts.py @@ -0,0 +1,31 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +from discover.fetchers.db.db_access import DbAccess + + +class DbFetchAZNetworkHosts(DbAccess): + + def get(self, id): + query = """ + SELECT DISTINCT host, host AS id, configurations + FROM neutron.agents + WHERE agent_type = 'Metadata agent' + """ + results = self.get_objects_list(query, "host") + for r in results: + self.set_host_details(r) + return results + + def set_host_details(self, r): + config = json.loads(r["configurations"]) + r["ip_address"] = config["nova_metadata_ip"] + r["host_type"] = "Network Node" diff --git a/app/discover/fetchers/db/db_fetch_host_instances.py b/app/discover/fetchers/db/db_fetch_host_instances.py new file mode 100644 index 0000000..2245c4a --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_host_instances.py @@ -0,0 +1,15 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.db.db_fetch_instances import DbFetchInstances + +class DbFetchHostInstances(DbFetchInstances): + + def get(self, id): + return self.get_instances("host", id) diff --git a/app/discover/fetchers/db/db_fetch_host_network_agents.py b/app/discover/fetchers/db/db_fetch_host_network_agents.py new file mode 100644 index 0000000..c323573 --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_host_network_agents.py @@ -0,0 +1,35 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +from discover.fetchers.db.db_access import DbAccess +from utils.inventory_mgr import InventoryMgr + + +class DbFetchHostNetworkAgents(DbAccess): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.env_config = self.config.get_env_config() + + def get(self, id): + query = """ + SELECT * FROM {}.agents + WHERE host = %s + """.format(self.neutron_db) + host_id = id[:-1 * len("-network_agents")] + results = self.get_objects_list_for_id(query, "network_agent", host_id) + mechanism_drivers = self.env_config['mechanism_drivers'] + id_prefix = mechanism_drivers[0] if mechanism_drivers else 'network_agent' + for o in results: + o["configurations"] = json.loads(o["configurations"]) + o["name"] = o["binary"] + o['id'] = id_prefix + '-' + o['id'] + return results diff --git a/app/discover/fetchers/db/db_fetch_instances.py b/app/discover/fetchers/db/db_fetch_instances.py new file mode 100644 index 0000000..54c4114 --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_instances.py @@ -0,0 +1,60 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +from discover.fetchers.db.db_access import DbAccess + + +class DbFetchInstances(DbAccess): + def get_instance_data(self, instances): + instances_hash = {} + for doc in instances: + instances_hash[doc["id"]] = doc + + query = """ + SELECT DISTINCT i.uuid AS id, i.display_name AS name, + i.host AS host, host_ip AS ip_address, + network_info, project_id, + IF(p.name IS NULL, "Unknown", p.name) AS project + FROM nova.instances i + LEFT JOIN keystone.project p ON p.id = i.project_id + JOIN nova.instance_info_caches ic ON i.uuid = ic.instance_uuid + JOIN nova.compute_nodes cn ON i.node = cn.hypervisor_hostname + WHERE i.deleted = 0 + """ + results = self.get_objects_list(query, "instance") + for result in results: + id = result["id"] + if id not in instances_hash: + continue + self.build_instance_details(result) + doc = instances_hash[id] + doc.update(result) + + def build_instance_details(self, result): + network_info_str = result.pop("network_info", None) + result["network_info"] = json.loads(network_info_str) + + # add network as an array to allow constraint checking + # when building clique + networks = [] + for net in result["network_info"]: + if "network" not in net or "id" not in net["network"]: + continue + network_id = net["network"]["id"] + if network_id in networks: + continue + networks.append(network_id) + result["network"] = networks + + result["type"] = "instance" + result["parent_type"] = "instances_folder" + result["parent_id"] = result["host"] + "-instances" + result["in_project-" + result.pop("project", None)] = "1" diff --git a/app/discover/fetchers/db/db_fetch_oteps.py b/app/discover/fetchers/db/db_fetch_oteps.py new file mode 100644 index 0000000..9055c11 --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_oteps.py @@ -0,0 +1,81 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.cli.cli_access import CliAccess +from discover.fetchers.db.db_access import DbAccess +from utils.inventory_mgr import InventoryMgr +from utils.singleton import Singleton + + +class DbFetchOteps(DbAccess, CliAccess, metaclass=Singleton): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.port_re = re.compile("^\s*port (\d+): ([^(]+)( \(internal\))?$") + + def get(self, id): + vedge = self.inv.get_by_id(self.get_env(), id) + tunnel_type = None + if "configurations" not in vedge: + return [] + if "tunnel_types" not in vedge["configurations"]: + return [] + if not vedge["configurations"]["tunnel_types"]: + return [] + tunnel_type = vedge["configurations"]["tunnel_types"][0] + host_id = vedge["host"] + table_name = "neutron.ml2_" + tunnel_type + "_endpoints" + env_config = self.config.get_env_config() + distribution = env_config["distribution"] + if distribution == "Canonical-icehouse": + # for Icehouse, we only get IP address from the DB, so take the + # host IP address and from the host data in Mongo + host = self.inv.get_by_id(self.get_env(), host_id) + results = [{"host": host_id, "ip_address": host["ip_address"]}] + else: + results = self.get_objects_list_for_id( + """ + SELECT * + FROM {} + WHERE host = %s + """.format(table_name), + "vedge", host_id) + for doc in results: + doc["id"] = host_id + "-otep" + doc["name"] = doc["id"] + doc["host"] = host_id + doc["overlay_type"] = tunnel_type + doc["ports"] = vedge["tunnel_ports"] if "tunnel_ports" in vedge else [] + if "udp_port" not in doc: + doc["udp_port"] = "67" + self.get_vconnector(doc, host_id, vedge) + + return results + + # find matching vConnector by tunneling_ip of vEdge + # look for that IP address in ifconfig for the host + def get_vconnector(self, doc, host_id, vedge): + tunneling_ip = vedge["configurations"]["tunneling_ip"] + ifconfig_lines = self.run_fetch_lines("ifconfig", host_id) + interface = None + ip_string = " " * 10 + "inet addr:" + tunneling_ip + " " + vconnector = None + for l in ifconfig_lines: + if l.startswith(" "): + if interface and l.startswith(ip_string): + vconnector = interface + break + else: + if " " in l: + interface = l[:l.index(" ")] + + if vconnector: + doc["vconnector"] = vconnector diff --git a/app/discover/fetchers/db/db_fetch_port.py b/app/discover/fetchers/db/db_fetch_port.py new file mode 100644 index 0000000..2cb814a --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_port.py @@ -0,0 +1,34 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.db.db_access import DbAccess +from utils.inventory_mgr import InventoryMgr + + +class DbFetchPort(DbAccess): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.env_config = self.config.get_env_config() + + def get(self, id=None): + query = """SELECT * FROM {}.ports where network_id = %s""" \ + .format(self.neutron_db) + return self.get_objects_list_for_id(query, "port", id) + + def get_id(self, id=None): + query = """SELECT id FROM {}.ports where network_id = %s""" \ + .format(self.neutron_db) + result = self.get_objects_list_for_id(query, "port", id) + return result[0]['id'] if result != [] else None + + def get_id_by_field(self, id, search=''): + query = """SELECT id FROM neutron.ports where network_id = %s AND """ + search + result = self.get_objects_list_for_id(query, "port", id) + return result[0]['id'] if result != [] else None
\ No newline at end of file diff --git a/app/discover/fetchers/db/db_fetch_vedges_ovs.py b/app/discover/fetchers/db/db_fetch_vedges_ovs.py new file mode 100644 index 0000000..24cc9f8 --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_vedges_ovs.py @@ -0,0 +1,178 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +import re + +from discover.fetchers.cli.cli_access import CliAccess +from discover.fetchers.db.db_access import DbAccess +from utils.inventory_mgr import InventoryMgr +from utils.singleton import Singleton + + +class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.port_re = re.compile("^\s*port (\d+): ([^(]+)( \(internal\))?$") + self.port_line_header_prefix = " " * 8 + "Port " + + def get(self, id): + host_id = id[:id.rindex('-')] + results = self.get_objects_list_for_id( + """ + SELECT * + FROM {}.agents + WHERE host = %s AND agent_type = 'Open vSwitch agent' + """.format(self.neutron_db), + "vedge", host_id) + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error("unable to find host in inventory: %s", host_id) + return [] + host_types = host["host_type"] + if "Network" not in host_types and "Compute" not in host_types: + return [] + vsctl_lines = self.run_fetch_lines("ovs-vsctl show", host["id"]) + ports = self.fetch_ports(host, vsctl_lines) + for doc in results: + doc["name"] = doc["host"] + "-OVS" + doc["configurations"] = json.loads(doc["configurations"]) + doc["ports"] = ports + doc["tunnel_ports"] = self.get_overlay_tunnels(doc, vsctl_lines) + return results + + def fetch_ports(self, host, vsctl_lines): + host_types = host["host_type"] + if "Network" not in host_types and "Compute" not in host_types: + return {} + ports = self.fetch_ports_from_dpctl(host["id"]) + self.fetch_port_tags_from_vsctl(vsctl_lines, ports) + return ports + + def fetch_ports_from_dpctl(self, host_id): + cmd = "ovs-dpctl show" + lines = self.run_fetch_lines(cmd, host_id) + ports = {} + for l in lines: + port_matches = self.port_re.match(l) + if not port_matches: + continue + port = {} + id = port_matches.group(1) + name = port_matches.group(2) + is_internal = port_matches.group(3) == " (internal)" + port["internal"] = is_internal + port["id"] = id + port["name"] = name + ports[name] = port + return ports + + # from ovs-vsctl, fetch tags of ports + # example format of ovs-vsctl output for a specific port: + # Port "tap9f94d28e-7b" + # tag: 5 + # Interface "tap9f94d28e-7b" + # type: internal + def fetch_port_tags_from_vsctl(self, vsctl_lines, ports): + port = None + for l in vsctl_lines: + if l.startswith(self.port_line_header_prefix): + port = None + port_name = l[len(self.port_line_header_prefix):] + # remove quotes from port name + if '"' in port_name: + port_name = port_name[1:][:-1] + if port_name in ports: + port = ports[port_name] + continue + if not port: + continue + if l.startswith(" " * 12 + "tag: "): + port["tag"] = l[l.index(":") + 2:] + ports[port["name"]] = port + return ports + + def get_overlay_tunnels(self, doc, vsctl_lines): + if doc["agent_type"] != "Open vSwitch agent": + return {} + if "tunneling_ip" not in doc["configurations"]: + return {} + if not doc["configurations"]["tunneling_ip"]: + self.get_bridge_pnic(doc) + return {} + + # read the 'br-tun' interface ports + # this will be used later in the OTEP + tunnel_bridge_header = " " * 4 + "Bridge br-tun" + try: + br_tun_loc = vsctl_lines.index(tunnel_bridge_header) + except ValueError: + return [] + lines = vsctl_lines[br_tun_loc + 1:] + tunnel_ports = {} + port = None + for l in lines: + # if we have only 4 or less spaces in the beginng, + # the br-tun section ended so return + if not l.startswith(" " * 5): + break + if l.startswith(self.port_line_header_prefix): + if port: + tunnel_ports[port["name"]] = port + name = l[len(self.port_line_header_prefix):].strip('" ') + port = {"name": name} + elif port and l.startswith(" " * 12 + "Interface "): + interface = l[10 + len("Interface ") + 1:].strip('" ') + port["interface"] = interface + elif port and l.startswith(" " * 16): + colon_pos = l.index(":") + attr = l[:colon_pos].strip() + val = l[colon_pos + 2:].strip('" ') + if attr == "options": + opts = val.strip('{}') + val = {} + for opt in opts.split(", "): + opt_name = opt[:opt.index("=")] + opt_val = opt[opt.index("=") + 1:].strip('" ') + val[opt_name] = opt_val + port[attr] = val + if port: + tunnel_ports[port["name"]] = port + return tunnel_ports + + def get_bridge_pnic(self, doc): + conf = doc["configurations"] + if "bridge_mappings" not in conf or not conf["bridge_mappings"]: + return + for v in conf["bridge_mappings"].values(): br = v + ifaces_list_lines = self.run_fetch_lines("ovs-vsctl list-ifaces " + br, + doc["host"]) + br_pnic_postfix = br + "--br-" + interface = "" + for l in ifaces_list_lines: + if l.startswith(br_pnic_postfix): + interface = l[len(br_pnic_postfix):] + break + if not interface: + return + doc["pnic"] = interface + # add port ID to pNIC + pnic = self.inv.find_items({ + "environment": self.get_env(), + "type": "pnic", + "host": doc["host"], + "name": interface + }, get_single=True) + if not pnic: + return + port = doc["ports"][interface] + pnic["port_id"] = port["id"] + self.inv.set(pnic) diff --git a/app/discover/fetchers/db/db_fetch_vedges_vpp.py b/app/discover/fetchers/db/db_fetch_vedges_vpp.py new file mode 100644 index 0000000..a1c659e --- /dev/null +++ b/app/discover/fetchers/db/db_fetch_vedges_vpp.py @@ -0,0 +1,56 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_access import CliAccess +from discover.fetchers.db.db_access import DbAccess +from utils.inventory_mgr import InventoryMgr +from utils.singleton import Singleton + + +class DbFetchVedgesVpp(DbAccess, CliAccess, metaclass=Singleton): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def get(self, id): + host_id = id[:id.rindex('-')] + vedge = { + 'host': host_id, + 'id': host_id + '-VPP', + 'name': 'VPP-' + host_id, + 'agent_type': 'VPP' + } + ver = self.run_fetch_lines('vppctl show ver', host_id) + if ver: + ver = ver[0] + vedge['binary'] = ver[:ver.index(' ', ver.index(' ') + 1)] + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error("unable to find host in inventory: %s", host_id) + return [] + host_types = host["host_type"] + if "Network" not in host_types and "Compute" not in host_types: + return [] + interfaces = self.run_fetch_lines('vppctl show int', host_id) + vedge['ports'] = self.fetch_ports(interfaces) + return [vedge] + + def fetch_ports(self, interfaces): + ports = {} + for i in interfaces: + if not i or i.startswith(' '): + continue + parts = i.split() + port = { + 'id': parts[1], + 'state': parts[2], + 'name': parts[0] + } + ports[port['name']] = port + return ports diff --git a/app/discover/fetchers/folder_fetcher.py b/app/discover/fetchers/folder_fetcher.py new file mode 100644 index 0000000..e7bb1fa --- /dev/null +++ b/app/discover/fetchers/folder_fetcher.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetcher import Fetcher +from utils.string_utils import jsonify + + +class FolderFetcher(Fetcher): + def __init__(self, types_name, parent_type, text="", create_folder=True): + super(FolderFetcher, self).__init__() + self.types_name = types_name + self.parent_type = parent_type + self.text = text + self.create_folder = create_folder + if not self.text: + self.text = self.types_name.capitalize() + + def get(self, id): + oid = id + "-" + self.types_name + root_obj = { + "id": oid, + "create_object": self.create_folder, + "name": oid, + "text": self.text, + "type": self.types_name + "_folder", + "parent_id": id, + "parent_type": self.parent_type + } + return jsonify([root_obj]) |