diff options
Diffstat (limited to 'app/discover/fetchers/api')
| -rw-r--r-- | app/discover/fetchers/api/__init__.py | 9 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_access.py | 195 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_availability_zones.py | 56 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_end_points.py | 35 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_host_instances.py | 59 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_network.py | 76 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_networks.py | 86 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_port.py | 60 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_ports.py | 55 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_project_hosts.py | 144 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_projects.py | 66 | ||||
| -rw-r--r-- | app/discover/fetchers/api/api_fetch_regions.py | 51 |
12 files changed, 892 insertions, 0 deletions
diff --git a/app/discover/fetchers/api/__init__.py b/app/discover/fetchers/api/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/api/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/api/api_access.py b/app/discover/fetchers/api/api_access.py new file mode 100644 index 0000000..89eeb34 --- /dev/null +++ b/app/discover/fetchers/api/api_access.py @@ -0,0 +1,195 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import calendar +import re +import requests +import time + +from discover.configuration import Configuration +from discover.fetcher import Fetcher +from utils.string_utils import jsonify + + +class ApiAccess(Fetcher): + subject_token = None + initialized = False + regions = {} + config = None + api_config = None + + host = "" + base_url = "" + admin_token = "" + tokens = {} + admin_endpoint = "" + admin_project = None + auth_response = None + + alternative_services = { + "neutron": ["quantum"] + } + + # identitity API v2 version with admin token + def __init__(self): + super(ApiAccess, self).__init__() + if ApiAccess.initialized: + return + ApiAccess.config = Configuration() + ApiAccess.api_config = ApiAccess.config.get("OpenStack") + host = ApiAccess.api_config["host"] + ApiAccess.host = host + port = ApiAccess.api_config["port"] + if not (host and port): + raise ValueError('Missing definition of host or port ' + + 'for OpenStack API access') + ApiAccess.base_url = "http://" + host + ":" + port + ApiAccess.admin_token = ApiAccess.api_config["admin_token"] + ApiAccess.admin_project = ApiAccess.api_config["admin_project"] \ + if "admin_project" in ApiAccess.api_config \ + else 'admin' + ApiAccess.admin_endpoint = "http://" + host + ":" + "35357" + + token = self.v2_auth_pwd(ApiAccess.admin_project) + if not token: + raise ValueError("Authentication failed. Failed to obtain token") + else: + self.subject_token = token + + @staticmethod + def parse_time(time_str): + try: + time_struct = time.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + try: + time_struct = time.strptime(time_str, + "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + return None + return time_struct + + # try to use existing token, if it did not expire + def get_existing_token(self, project_id): + try: + token_details = ApiAccess.tokens[project_id] + except KeyError: + return None + token_expiry = token_details["expires"] + token_expiry_time_struct = self.parse_time(token_expiry) + if not token_expiry_time_struct: + return None + token_expiry_time = token_details["token_expiry_time"] + now = time.time() + if now > token_expiry_time: + # token has expired + ApiAccess.tokens.pop(project_id) + return None + return token_details + + def v2_auth(self, project_id, headers, post_body): + subject_token = self.get_existing_token(project_id) + if subject_token: + return subject_token + req_url = ApiAccess.base_url + "/v2.0/tokens" + response = requests.post(req_url, json=post_body, headers=headers) + ApiAccess.auth_response = response.json() + if 'error' in self.auth_response: + e = self.auth_response['error'] + self.log.error(str(e['code']) + ' ' + e['title'] + ': ' + + e['message'] + ", URL: " + req_url) + return None + try: + token_details = ApiAccess.auth_response["access"]["token"] + except KeyError: + # assume authentication failed + return None + token_expiry = token_details["expires"] + token_expiry_time_struct = self.parse_time(token_expiry) + if not token_expiry_time_struct: + return None + token_expiry_time = calendar.timegm(token_expiry_time_struct) + token_details["token_expiry_time"] = token_expiry_time + ApiAccess.tokens[project_id] = token_details + return token_details + + def v2_auth_pwd(self, project): + user = ApiAccess.api_config["user"] + pwd = ApiAccess.api_config["pwd"] + post_body = { + "auth": { + "passwordCredentials": { + "username": user, + "password": pwd + } + } + } + if project is not None: + post_body["auth"]["tenantName"] = project + project_id = project + else: + project_id = "" + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json; charset=UTF-8' + } + return self.v2_auth(project_id, headers, post_body) + + def get_rel_url(self, relative_url, headers): + req_url = ApiAccess.base_url + relative_url + return self.get_url(req_url, headers) + + def get_url(self, req_url, headers): + response = requests.get(req_url, headers=headers) + if response.status_code != requests.codes.ok: + # some error happened + if "reason" in response: + msg = ", reason: {}".format(response.reason) + else: + msg = ", response: {}".format(response.text) + self.log.error("req_url: {} {}".format(req_url, msg)) + return response + ret = response.json() + return ret + + def get_region_url(self, region_name, service): + if region_name not in self.regions: + return None + region = self.regions[region_name] + s = self.get_service_region_endpoints(region, service) + if not s: + return None + orig_url = s["adminURL"] + # replace host name with the host found in config + url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url) + return url + + # like get_region_url(), but remove everything starting from the "/v2" + def get_region_url_nover(self, region, service): + full_url = self.get_region_url(region, service) + if not full_url: + self.log.error("could not find region URL for region: " + region) + exit() + url = re.sub(r":([0-9]+)/v[2-9].*", r":\1", full_url) + return url + + def get_catalog(self, pretty): + return jsonify(self.regions, pretty) + + # find the endpoints for a given service name, + # considering also alternative service names + def get_service_region_endpoints(self, region, service): + alternatives = [service] + endpoints = region["endpoints"] + if service in self.alternative_services: + alternatives.extend(self.alternative_services[service]) + for sname in alternatives: + if sname in endpoints: + return endpoints[sname] + return None + diff --git a/app/discover/fetchers/api/api_fetch_availability_zones.py b/app/discover/fetchers/api/api_fetch_availability_zones.py new file mode 100644 index 0000000..196893b --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_availability_zones.py @@ -0,0 +1,56 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess + + +class ApiFetchAvailabilityZones(ApiAccess): + def __init__(self): + super(ApiFetchAvailabilityZones, self).__init__() + + def get(self, project_id): + token = self.v2_auth_pwd(project_id) + if not token: + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_for_region(project_id, region, token)) + return ret + + def get_for_region(self, project, region, token): + # we use os-availability-zone/detail rather than os-availability-zone, + # because the later does not inclde the "internal" zone in the results + endpoint = self.get_region_url_nover(region, "nova") + req_url = endpoint + "/v2/" + token["tenant"]["id"] + \ + "/os-availability-zone/detail" + headers = { + "X-Auth-Project-Id": project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if "status" in response and int(response["status"]) != 200: + return [] + ret = [] + if "availabilityZoneInfo" not in response: + return [] + azs = response["availabilityZoneInfo"] + if not azs: + return [] + for doc in azs: + doc["id"] = doc["zoneName"] + doc["name"] = doc.pop("zoneName") + doc["master_parent_type"] = "region" + doc["master_parent_id"] = region + doc["parent_type"] = "availability_zones_folder" + doc["parent_id"] = region + "-availability_zones" + doc["parent_text"] = "Availability Zones" + doc["available"] = doc["zoneState"]["available"] + doc.pop("zoneState") + ret.append(doc) + return ret diff --git a/app/discover/fetchers/api/api_fetch_end_points.py b/app/discover/fetchers/api/api_fetch_end_points.py new file mode 100644 index 0000000..9471c7e --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_end_points.py @@ -0,0 +1,35 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# fetch the end points for a given project (tenant) +# return list of regions, to allow further recursive scanning + +from discover.fetchers.api.api_access import ApiAccess + + +class ApiFetchEndPoints(ApiAccess): + + def get(self, project_id): + if project_id != "admin": + return [] # XXX currently having problems authenticating to other tenants + self.v2_auth_pwd(project_id) + + environment = ApiAccess.config.get_env_name() + regions = [] + services = ApiAccess.auth_response['access']['serviceCatalog'] + endpoints = [] + for s in services: + if s["type"] != "identity": + continue + e = s["endpoints"][0] + e["environment"] = environment + e["project"] = project_id + e["type"] = "endpoint" + endpoints.append(e) + return endpoints diff --git a/app/discover/fetchers/api/api_fetch_host_instances.py b/app/discover/fetchers/api/api_fetch_host_instances.py new file mode 100644 index 0000000..56cffda --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_host_instances.py @@ -0,0 +1,59 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from discover.fetchers.db.db_access import DbAccess +from discover.fetchers.db.db_fetch_instances import DbFetchInstances +from utils.inventory_mgr import InventoryMgr +from utils.singleton import Singleton + + +class ApiFetchHostInstances(ApiAccess, DbAccess, metaclass=Singleton): + def __init__(self): + super(ApiFetchHostInstances, self).__init__() + self.inv = InventoryMgr() + self.endpoint = ApiAccess.base_url.replace(":5000", ":8774") + self.projects = None + self.db_fetcher = DbFetchInstances() + + def get_projects(self): + if not self.projects: + projects_list = self.inv.get(self.get_env(), "project", None) + self.projects = [p["name"] for p in projects_list] + + def get(self, id): + self.get_projects() + host_id = id[:id.rindex("-")] + host = self.inv.get_by_id(self.get_env(), host_id) + if not host or "Compute" not in host.get("host_type", ""): + return [] + instances_found = self.get_instances_from_api(host_id) + self.db_fetcher.get_instance_data(instances_found) + return instances_found + + def get_instances_from_api(self, host_name): + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + tenant_id = token["tenant"]["id"] + req_url = self.endpoint + "/v2/" + tenant_id + \ + "/os-hypervisors/" + host_name + "/servers" + response = self.get_url(req_url, {"X-Auth-Token": token["id"]}) + ret = [] + if not "hypervisors" in response: + return [] + if not "servers" in response["hypervisors"][0]: + return [] + for doc in response["hypervisors"][0]["servers"]: + doc["id"] = doc["uuid"] + doc["host"] = host_name + doc["local_name"] = doc.pop("name") + ret.append(doc) + self.log.info("found %s instances for host: %s", str(len(ret)), host_name) + return ret diff --git a/app/discover/fetchers/api/api_fetch_network.py b/app/discover/fetchers/api/api_fetch_network.py new file mode 100644 index 0000000..889b8a5 --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_network.py @@ -0,0 +1,76 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from utils.inventory_mgr import InventoryMgr + + +class ApiFetchNetwork(ApiAccess): + def __init__(self): + super(ApiFetchNetwork, self).__init__() + self.inv = InventoryMgr() + + def get(self, project_id): + # use project admin credentials, to be able to fetch all networks + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + # TODO: refactor legacy code (Unresolved reference - self.get_for_region) + ret.extend(self.get_for_region(region, token, project_id)) + return ret + + def get_network(self, region, token, subnet_id): + endpoint = self.get_region_url_nover(region, "neutron") + + # get target network network document + req_url = endpoint + "/v2.0/networks/" + subnet_id + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not "network" in response: + return [] + network = response["network"] + subnets = network['subnets'] + + # get subnets documents. + subnets_hash = {} + cidrs = [] + subnet_ids = [] + for subnet_id in subnets: + req_url = endpoint + "/v2.0/subnets/" + subnet_id + response = self.get_url(req_url, headers) + if "subnet" in response: + # create a hash subnets, to allow easy locating of subnets + subnet = response["subnet"] + subnets_hash[subnet["name"]] = subnet + cidrs.append(subnet["cidr"]) + subnet_ids.append(subnet["id"]) + + network["subnets"] = subnets_hash + network["cidrs"] = cidrs + network["subnet_ids"] = subnet_ids + + network["master_parent_type"] = "project" + network["master_parent_id"] = network["tenant_id"] + network["parent_type"] = "networks_folder" + network["parent_id"] = network["tenant_id"] + "-networks" + network["parent_text"] = "Networks" + # set the 'network' attribute for network objects to the name of network, + # to allow setting constraint on network when creating network clique + network['network'] = network["id"] + # get the project name + project = self.inv.get_by_id(self.get_env(), network["tenant_id"]) + if project: + network["project"] = project["name"] + + return network diff --git a/app/discover/fetchers/api/api_fetch_networks.py b/app/discover/fetchers/api/api_fetch_networks.py new file mode 100644 index 0000000..4b70f65 --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_networks.py @@ -0,0 +1,86 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from utils.inventory_mgr import InventoryMgr + + +class ApiFetchNetworks(ApiAccess): + def __init__(self): + super(ApiFetchNetworks, self).__init__() + self.inv = InventoryMgr() + + def get(self, project_id=None): + # use project admin credentials, to be able to fetch all networks + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_networks(region, token)) + return ret + + def get_networks(self, region, token): + endpoint = self.get_region_url_nover(region, "neutron") + req_url = endpoint + "/v2.0/networks" + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not "networks" in response: + return [] + networks = response["networks"] + req_url = endpoint + "/v2.0/subnets" + response = self.get_url(req_url, headers) + subnets_hash = {} + if "subnets" in response: + # create a hash subnets, to allow easy locating of subnets + subnets = response["subnets"] + for s in subnets: + subnets_hash[s["id"]] = s + for doc in networks: + doc["master_parent_type"] = "project" + project_id = doc["tenant_id"] + if not project_id: + # find project ID of admin project + project = self.inv.get_by_field(self.get_env(), + "project", "name", + self.admin_project, + get_single=True) + if not project: + self.log.error("failed to find admin project in DB") + project_id = project["id"] + doc["master_parent_id"] = project_id + doc["parent_type"] = "networks_folder" + doc["parent_id"] = project_id + "-networks" + doc["parent_text"] = "Networks" + # set the 'network' attribute for network objects to the name of network, + # to allow setting constraint on network when creating network clique + doc['network'] = doc["id"] + # get the project name + project = self.inv.get_by_id(self.get_env(), project_id) + if project: + doc["project"] = project["name"] + subnets_details = {} + cidrs = [] + subnet_ids = [] + for s in doc["subnets"]: + try: + subnet = subnets_hash[s] + cidrs.append(subnet["cidr"]) + subnet_ids.append(subnet["id"]) + subnets_details[subnet["name"]] = subnet + except KeyError: + pass + + doc["subnets"] = subnets_details + doc["cidrs"] = cidrs + doc["subnet_ids"] = subnet_ids + return networks diff --git a/app/discover/fetchers/api/api_fetch_port.py b/app/discover/fetchers/api/api_fetch_port.py new file mode 100644 index 0000000..f8d9eeb --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_port.py @@ -0,0 +1,60 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from utils.inventory_mgr import InventoryMgr + + +class ApiFetchPort(ApiAccess): + def __init__(self): + super(ApiFetchPort, self).__init__() + self.inv = InventoryMgr() + + def get(self, project_id): + if not project_id: + self.log.info("Get method needs ID parameter") + return [] + # use project admin credentials, to be able to fetch all ports + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + ret.append(self.get_port(region, token, project_id)) + if ret == []: + self.log.info("ApiFetchPort: Port not found.") + return ret + + def get_port(self, region, token, id): + endpoint = self.get_region_url_nover(region, "neutron") + req_url = endpoint + "/v2.0/ports/" + id + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not "port" in response: + return [] + + doc = response["port"] + doc["master_parent_type"] = "network" + doc["master_parent_id"] = doc["network_id"] + doc["parent_type"] = "ports_folder" + doc["parent_id"] = doc["network_id"] + "-ports" + doc["parent_text"] = "Ports" + # get the project name + net = self.inv.get_by_id(self.get_env(), doc["network_id"]) + if net: + doc["name"] = doc["mac_address"] + else: + doc["name"] = doc["id"] + project = self.inv.get_by_id(self.get_env(), doc["tenant_id"]) + if project: + doc["project"] = project["name"] + return doc diff --git a/app/discover/fetchers/api/api_fetch_ports.py b/app/discover/fetchers/api/api_fetch_ports.py new file mode 100644 index 0000000..f4c54a6 --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_ports.py @@ -0,0 +1,55 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess +from utils.inventory_mgr import InventoryMgr + + +class ApiFetchPorts(ApiAccess): + def __init__(self): + super(ApiFetchPorts, self).__init__() + self.inv = InventoryMgr() + + def get(self, project_id): + # use project admin credentials, to be able to fetch all ports + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_ports_for_region(region, token)) + return ret + + def get_ports_for_region(self, region, token): + endpoint = self.get_region_url_nover(region, "neutron") + req_url = endpoint + "/v2.0/ports" + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not "ports" in response: + return [] + ports = response["ports"] + for doc in ports: + doc["master_parent_type"] = "network" + doc["master_parent_id"] = doc["network_id"] + doc["parent_type"] = "ports_folder" + doc["parent_id"] = doc["network_id"] + "-ports" + doc["parent_text"] = "Ports" + # get the project name + net = self.inv.get_by_id(self.get_env(), doc["network_id"]) + if net: + doc["name"] = doc["mac_address"] + else: + doc["name"] = doc["id"] + project = self.inv.get_by_id(self.get_env(), doc["tenant_id"]) + if project: + doc["project"] = project["name"] + return ports diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py new file mode 100644 index 0000000..7dc262e --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_project_hosts.py @@ -0,0 +1,144 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +from discover.fetchers.api.api_access import ApiAccess +from discover.fetchers.db.db_access import DbAccess + + +class ApiFetchProjectHosts(ApiAccess, DbAccess): + def __init__(self): + super(ApiFetchProjectHosts, self).__init__() + + def get(self, project_id): + if project_id != self.admin_project: + # do not scan hosts except under project 'admin' + return [] + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_for_region(region, token)) + return ret + + def get_for_region(self, region, token): + endpoint = self.get_region_url(region, "nova") + ret = [] + if not token: + return [] + req_url = endpoint + "/os-availability-zone/detail" + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if "status" in response and int(response["status"]) != 200: + return [] + az_info = response["availabilityZoneInfo"] + hosts = {} + for doc in az_info: + az_hosts = self.get_hosts_from_az(doc) + for h in az_hosts: + if h["name"] in hosts: + # merge host_type data between AZs + existing_entry = hosts[h["name"]] + for t in h["host_type"]: + self.add_host_type(existing_entry, t, doc['zoneName']) + else: + hosts[h["name"]] = h + ret.append(h) + # get os_id for hosts using the os-hypervisors API call + req_url = endpoint + "/os-hypervisors" + response = self.get_url(req_url, headers) + if "status" in response and int(response["status"]) != 200: + return ret + if "hypervisors" not in response: + return ret + for h in response["hypervisors"]: + hvname = h["hypervisor_hostname"] + if '.' in hvname and hvname not in hosts: + hostname = hvname[:hvname.index('.')] + else: + hostname = hvname + try: + doc = hosts[hostname] + except KeyError: + # TBD - add error output + continue + doc["os_id"] = str(h["id"]) + self.fetch_compute_node_ip_address(doc, hvname) + # get more network nodes details + self.fetch_network_node_details(ret) + return ret + + def get_hosts_from_az(self, az): + ret = [] + for h in az["hosts"]: + doc = self.get_host_details(az, h) + ret.append(doc) + return ret + + def get_host_details(self, az, h): + # for hosts we use the name + services = az["hosts"][h] + doc = { + "id": h, + "host": h, + "name": h, + "zone": az["zoneName"], + "parent_type": "availability_zone", + "parent_id": az["zoneName"], + "services": services, + "host_type": [] + } + if "nova-conductor" in services: + s = services["nova-conductor"] + if s["available"] and s["active"]: + self.add_host_type(doc, "Controller", az['zoneName']) + if "nova-compute" in services: + s = services["nova-compute"] + if s["available"] and s["active"]: + self.add_host_type(doc, "Compute", az['zoneName']) + return doc + + # fetch more details of network nodes from neutron.agents table + def fetch_network_node_details(self, docs): + hosts = {} + for doc in docs: + hosts[doc["host"]] = doc + query = """ + SELECT DISTINCT host, host AS id, configurations + FROM {}.agents + WHERE agent_type IN ('Metadata agent', 'DHCP agent', 'L3 agent') + """.format(self.neutron_db) + results = self.get_objects_list(query, "") + for r in results: + host = hosts[r["host"]] + host["config"] = json.loads(r["configurations"]) + self.add_host_type(host, "Network", '') + + # fetch ip_address from nova.compute_nodes table if possible + def fetch_compute_node_ip_address(self, doc, h): + query = """ + SELECT host_ip AS ip_address + FROM nova.compute_nodes + WHERE hypervisor_hostname = %s + """ + results = self.get_objects_list_for_id(query, "", h) + for db_row in results: + doc.update(db_row) + + def add_host_type(self, doc, type, zone): + if not type in doc["host_type"]: + doc["host_type"].append(type) + if type == 'Compute': + doc['zone'] = zone + doc['parent_id'] = zone diff --git a/app/discover/fetchers/api/api_fetch_projects.py b/app/discover/fetchers/api/api_fetch_projects.py new file mode 100644 index 0000000..4ef8083 --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_projects.py @@ -0,0 +1,66 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess + + +class ApiFetchProjects(ApiAccess): + def __init__(self): + super(ApiFetchProjects, self).__init__() + + def get(self, project_id): + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + if not self.regions: + self.log.error('No regions found') + return [] + ret = [] + for region in self.regions: + ret.extend(self.get_for_region(region, token)) + projects_for_user = self.get_projects_for_api_user(region, token) + return [p for p in ret if p['name'] in projects_for_user] \ + if projects_for_user else ret + + def get_projects_for_api_user(self, region, token): + if not token: + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + endpoint = self.get_region_url_nover(region, "keystone") + headers = { + 'X-Auth-Project-Id': self.admin_project, + 'X-Auth-Token': token['id'] + } + # get the list of projects accessible by the admin user + req_url = endpoint + '/v3/projects' + response = self.get_url(req_url, headers) + if not response or 'projects' not in response: + return None + response = [p['name'] for p in response['projects']] + return response + + def get_for_region(self, region, token): + endpoint = self.get_region_url_nover(region, "keystone") + req_url = endpoint + "/v2.0/tenants" + headers = { + "X-Auth-Project-Id": self.admin_project, + "X-Auth-Token": token["id"] + } + response = self.get_url(req_url, headers) + if not isinstance(response, dict): + self.log.error('invalid response to /tenants request: not dict') + return [] + tenants_list = response.get("tenants", []) + if not isinstance(tenants_list, list): + self.log.error('invalid response to /tenants request: ' + 'tenants value is n ot a list') + return [] + response = [t for t in tenants_list if t.get("name", "") != "services"] + return response diff --git a/app/discover/fetchers/api/api_fetch_regions.py b/app/discover/fetchers/api/api_fetch_regions.py new file mode 100644 index 0000000..dcc558f --- /dev/null +++ b/app/discover/fetchers/api/api_fetch_regions.py @@ -0,0 +1,51 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.api.api_access import ApiAccess + + +class ApiFetchRegions(ApiAccess): + def __init__(self): + super(ApiFetchRegions, self).__init__() + self.endpoint = ApiAccess.base_url + + def get(self, project_id): + token = self.v2_auth_pwd(self.admin_project) + if not token: + return [] + # the returned authentication response contains the list of end points + # and regions + service_catalog = ApiAccess.auth_response.get('access', {}).get('serviceCatalog') + if not service_catalog: + return [] + env = self.get_env() + ret = [] + NULL_REGION = "No-Region" + for service in service_catalog: + for e in service["endpoints"]: + if "region" in e: + region_name = e.pop("region") + region_name = region_name if region_name else NULL_REGION + else: + region_name = NULL_REGION + if region_name in self.regions.keys(): + region = self.regions[region_name] + else: + region = { + "id": region_name, + "name": region_name, + "endpoints": {} + } + ApiAccess.regions[region_name] = region + region["parent_type"] = "regions_folder" + region["parent_id"] = env + "-regions" + e["service_type"] = service["type"] + region["endpoints"][service["name"]] = e + ret.extend(list(ApiAccess.regions.values())) + return ret |