summaryrefslogtreecommitdiffstats
path: root/app/discover/fetchers/api
diff options
context:
space:
mode:
authorYaron Yogev <yaronyogev@gmail.com>2017-07-27 09:02:54 +0300
committerYaron Yogev <yaronyogev@gmail.com>2017-07-27 14:56:25 +0300
commit7e83d0876ddb84a45e130eeba28bc40ef53c074b (patch)
tree47d76239ae7658d87c66abd142df92709427e7dd /app/discover/fetchers/api
parent378ecbd8947589b9cbb39013a0c2e2aa201e03bd (diff)
Calipso initial release for OPNFV
Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88 Signed-off-by: Yaron Yogev <yaronyogev@gmail.com>
Diffstat (limited to 'app/discover/fetchers/api')
-rw-r--r--app/discover/fetchers/api/__init__.py9
-rw-r--r--app/discover/fetchers/api/api_access.py195
-rw-r--r--app/discover/fetchers/api/api_fetch_availability_zones.py56
-rw-r--r--app/discover/fetchers/api/api_fetch_end_points.py35
-rw-r--r--app/discover/fetchers/api/api_fetch_host_instances.py59
-rw-r--r--app/discover/fetchers/api/api_fetch_network.py76
-rw-r--r--app/discover/fetchers/api/api_fetch_networks.py86
-rw-r--r--app/discover/fetchers/api/api_fetch_port.py60
-rw-r--r--app/discover/fetchers/api/api_fetch_ports.py55
-rw-r--r--app/discover/fetchers/api/api_fetch_project_hosts.py144
-rw-r--r--app/discover/fetchers/api/api_fetch_projects.py66
-rw-r--r--app/discover/fetchers/api/api_fetch_regions.py51
12 files changed, 892 insertions, 0 deletions
diff --git a/app/discover/fetchers/api/__init__.py b/app/discover/fetchers/api/__init__.py
new file mode 100644
index 0000000..b0637e9
--- /dev/null
+++ b/app/discover/fetchers/api/__init__.py
@@ -0,0 +1,9 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
diff --git a/app/discover/fetchers/api/api_access.py b/app/discover/fetchers/api/api_access.py
new file mode 100644
index 0000000..89eeb34
--- /dev/null
+++ b/app/discover/fetchers/api/api_access.py
@@ -0,0 +1,195 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+import calendar
+import re
+import requests
+import time
+
+from discover.configuration import Configuration
+from discover.fetcher import Fetcher
+from utils.string_utils import jsonify
+
+
+class ApiAccess(Fetcher):
+ subject_token = None
+ initialized = False
+ regions = {}
+ config = None
+ api_config = None
+
+ host = ""
+ base_url = ""
+ admin_token = ""
+ tokens = {}
+ admin_endpoint = ""
+ admin_project = None
+ auth_response = None
+
+ alternative_services = {
+ "neutron": ["quantum"]
+ }
+
+ # identitity API v2 version with admin token
+ def __init__(self):
+ super(ApiAccess, self).__init__()
+ if ApiAccess.initialized:
+ return
+ ApiAccess.config = Configuration()
+ ApiAccess.api_config = ApiAccess.config.get("OpenStack")
+ host = ApiAccess.api_config["host"]
+ ApiAccess.host = host
+ port = ApiAccess.api_config["port"]
+ if not (host and port):
+ raise ValueError('Missing definition of host or port ' +
+ 'for OpenStack API access')
+ ApiAccess.base_url = "http://" + host + ":" + port
+ ApiAccess.admin_token = ApiAccess.api_config["admin_token"]
+ ApiAccess.admin_project = ApiAccess.api_config["admin_project"] \
+ if "admin_project" in ApiAccess.api_config \
+ else 'admin'
+ ApiAccess.admin_endpoint = "http://" + host + ":" + "35357"
+
+ token = self.v2_auth_pwd(ApiAccess.admin_project)
+ if not token:
+ raise ValueError("Authentication failed. Failed to obtain token")
+ else:
+ self.subject_token = token
+
+ @staticmethod
+ def parse_time(time_str):
+ try:
+ time_struct = time.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ")
+ except ValueError:
+ try:
+ time_struct = time.strptime(time_str,
+ "%Y-%m-%dT%H:%M:%S.%fZ")
+ except ValueError:
+ return None
+ return time_struct
+
+ # try to use existing token, if it did not expire
+ def get_existing_token(self, project_id):
+ try:
+ token_details = ApiAccess.tokens[project_id]
+ except KeyError:
+ return None
+ token_expiry = token_details["expires"]
+ token_expiry_time_struct = self.parse_time(token_expiry)
+ if not token_expiry_time_struct:
+ return None
+ token_expiry_time = token_details["token_expiry_time"]
+ now = time.time()
+ if now > token_expiry_time:
+ # token has expired
+ ApiAccess.tokens.pop(project_id)
+ return None
+ return token_details
+
+ def v2_auth(self, project_id, headers, post_body):
+ subject_token = self.get_existing_token(project_id)
+ if subject_token:
+ return subject_token
+ req_url = ApiAccess.base_url + "/v2.0/tokens"
+ response = requests.post(req_url, json=post_body, headers=headers)
+ ApiAccess.auth_response = response.json()
+ if 'error' in self.auth_response:
+ e = self.auth_response['error']
+ self.log.error(str(e['code']) + ' ' + e['title'] + ': ' +
+ e['message'] + ", URL: " + req_url)
+ return None
+ try:
+ token_details = ApiAccess.auth_response["access"]["token"]
+ except KeyError:
+ # assume authentication failed
+ return None
+ token_expiry = token_details["expires"]
+ token_expiry_time_struct = self.parse_time(token_expiry)
+ if not token_expiry_time_struct:
+ return None
+ token_expiry_time = calendar.timegm(token_expiry_time_struct)
+ token_details["token_expiry_time"] = token_expiry_time
+ ApiAccess.tokens[project_id] = token_details
+ return token_details
+
+ def v2_auth_pwd(self, project):
+ user = ApiAccess.api_config["user"]
+ pwd = ApiAccess.api_config["pwd"]
+ post_body = {
+ "auth": {
+ "passwordCredentials": {
+ "username": user,
+ "password": pwd
+ }
+ }
+ }
+ if project is not None:
+ post_body["auth"]["tenantName"] = project
+ project_id = project
+ else:
+ project_id = ""
+ headers = {
+ 'Accept': 'application/json',
+ 'Content-Type': 'application/json; charset=UTF-8'
+ }
+ return self.v2_auth(project_id, headers, post_body)
+
+ def get_rel_url(self, relative_url, headers):
+ req_url = ApiAccess.base_url + relative_url
+ return self.get_url(req_url, headers)
+
+ def get_url(self, req_url, headers):
+ response = requests.get(req_url, headers=headers)
+ if response.status_code != requests.codes.ok:
+ # some error happened
+ if "reason" in response:
+ msg = ", reason: {}".format(response.reason)
+ else:
+ msg = ", response: {}".format(response.text)
+ self.log.error("req_url: {} {}".format(req_url, msg))
+ return response
+ ret = response.json()
+ return ret
+
+ def get_region_url(self, region_name, service):
+ if region_name not in self.regions:
+ return None
+ region = self.regions[region_name]
+ s = self.get_service_region_endpoints(region, service)
+ if not s:
+ return None
+ orig_url = s["adminURL"]
+ # replace host name with the host found in config
+ url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url)
+ return url
+
+ # like get_region_url(), but remove everything starting from the "/v2"
+ def get_region_url_nover(self, region, service):
+ full_url = self.get_region_url(region, service)
+ if not full_url:
+ self.log.error("could not find region URL for region: " + region)
+ exit()
+ url = re.sub(r":([0-9]+)/v[2-9].*", r":\1", full_url)
+ return url
+
+ def get_catalog(self, pretty):
+ return jsonify(self.regions, pretty)
+
+ # find the endpoints for a given service name,
+ # considering also alternative service names
+ def get_service_region_endpoints(self, region, service):
+ alternatives = [service]
+ endpoints = region["endpoints"]
+ if service in self.alternative_services:
+ alternatives.extend(self.alternative_services[service])
+ for sname in alternatives:
+ if sname in endpoints:
+ return endpoints[sname]
+ return None
+
diff --git a/app/discover/fetchers/api/api_fetch_availability_zones.py b/app/discover/fetchers/api/api_fetch_availability_zones.py
new file mode 100644
index 0000000..196893b
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_availability_zones.py
@@ -0,0 +1,56 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+
+
+class ApiFetchAvailabilityZones(ApiAccess):
+ def __init__(self):
+ super(ApiFetchAvailabilityZones, self).__init__()
+
+ def get(self, project_id):
+ token = self.v2_auth_pwd(project_id)
+ if not token:
+ return []
+ ret = []
+ for region in self.regions:
+ ret.extend(self.get_for_region(project_id, region, token))
+ return ret
+
+ def get_for_region(self, project, region, token):
+ # we use os-availability-zone/detail rather than os-availability-zone,
+ # because the later does not inclde the "internal" zone in the results
+ endpoint = self.get_region_url_nover(region, "nova")
+ req_url = endpoint + "/v2/" + token["tenant"]["id"] + \
+ "/os-availability-zone/detail"
+ headers = {
+ "X-Auth-Project-Id": project,
+ "X-Auth-Token": token["id"]
+ }
+ response = self.get_url(req_url, headers)
+ if "status" in response and int(response["status"]) != 200:
+ return []
+ ret = []
+ if "availabilityZoneInfo" not in response:
+ return []
+ azs = response["availabilityZoneInfo"]
+ if not azs:
+ return []
+ for doc in azs:
+ doc["id"] = doc["zoneName"]
+ doc["name"] = doc.pop("zoneName")
+ doc["master_parent_type"] = "region"
+ doc["master_parent_id"] = region
+ doc["parent_type"] = "availability_zones_folder"
+ doc["parent_id"] = region + "-availability_zones"
+ doc["parent_text"] = "Availability Zones"
+ doc["available"] = doc["zoneState"]["available"]
+ doc.pop("zoneState")
+ ret.append(doc)
+ return ret
diff --git a/app/discover/fetchers/api/api_fetch_end_points.py b/app/discover/fetchers/api/api_fetch_end_points.py
new file mode 100644
index 0000000..9471c7e
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_end_points.py
@@ -0,0 +1,35 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+# fetch the end points for a given project (tenant)
+# return list of regions, to allow further recursive scanning
+
+from discover.fetchers.api.api_access import ApiAccess
+
+
+class ApiFetchEndPoints(ApiAccess):
+
+ def get(self, project_id):
+ if project_id != "admin":
+ return [] # XXX currently having problems authenticating to other tenants
+ self.v2_auth_pwd(project_id)
+
+ environment = ApiAccess.config.get_env_name()
+ regions = []
+ services = ApiAccess.auth_response['access']['serviceCatalog']
+ endpoints = []
+ for s in services:
+ if s["type"] != "identity":
+ continue
+ e = s["endpoints"][0]
+ e["environment"] = environment
+ e["project"] = project_id
+ e["type"] = "endpoint"
+ endpoints.append(e)
+ return endpoints
diff --git a/app/discover/fetchers/api/api_fetch_host_instances.py b/app/discover/fetchers/api/api_fetch_host_instances.py
new file mode 100644
index 0000000..56cffda
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_host_instances.py
@@ -0,0 +1,59 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from discover.fetchers.db.db_access import DbAccess
+from discover.fetchers.db.db_fetch_instances import DbFetchInstances
+from utils.inventory_mgr import InventoryMgr
+from utils.singleton import Singleton
+
+
+class ApiFetchHostInstances(ApiAccess, DbAccess, metaclass=Singleton):
+ def __init__(self):
+ super(ApiFetchHostInstances, self).__init__()
+ self.inv = InventoryMgr()
+ self.endpoint = ApiAccess.base_url.replace(":5000", ":8774")
+ self.projects = None
+ self.db_fetcher = DbFetchInstances()
+
+ def get_projects(self):
+ if not self.projects:
+ projects_list = self.inv.get(self.get_env(), "project", None)
+ self.projects = [p["name"] for p in projects_list]
+
+ def get(self, id):
+ self.get_projects()
+ host_id = id[:id.rindex("-")]
+ host = self.inv.get_by_id(self.get_env(), host_id)
+ if not host or "Compute" not in host.get("host_type", ""):
+ return []
+ instances_found = self.get_instances_from_api(host_id)
+ self.db_fetcher.get_instance_data(instances_found)
+ return instances_found
+
+ def get_instances_from_api(self, host_name):
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ tenant_id = token["tenant"]["id"]
+ req_url = self.endpoint + "/v2/" + tenant_id + \
+ "/os-hypervisors/" + host_name + "/servers"
+ response = self.get_url(req_url, {"X-Auth-Token": token["id"]})
+ ret = []
+ if not "hypervisors" in response:
+ return []
+ if not "servers" in response["hypervisors"][0]:
+ return []
+ for doc in response["hypervisors"][0]["servers"]:
+ doc["id"] = doc["uuid"]
+ doc["host"] = host_name
+ doc["local_name"] = doc.pop("name")
+ ret.append(doc)
+ self.log.info("found %s instances for host: %s", str(len(ret)), host_name)
+ return ret
diff --git a/app/discover/fetchers/api/api_fetch_network.py b/app/discover/fetchers/api/api_fetch_network.py
new file mode 100644
index 0000000..889b8a5
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_network.py
@@ -0,0 +1,76 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class ApiFetchNetwork(ApiAccess):
+ def __init__(self):
+ super(ApiFetchNetwork, self).__init__()
+ self.inv = InventoryMgr()
+
+ def get(self, project_id):
+ # use project admin credentials, to be able to fetch all networks
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ ret = []
+ for region in self.regions:
+ # TODO: refactor legacy code (Unresolved reference - self.get_for_region)
+ ret.extend(self.get_for_region(region, token, project_id))
+ return ret
+
+ def get_network(self, region, token, subnet_id):
+ endpoint = self.get_region_url_nover(region, "neutron")
+
+ # get target network network document
+ req_url = endpoint + "/v2.0/networks/" + subnet_id
+ headers = {
+ "X-Auth-Project-Id": self.admin_project,
+ "X-Auth-Token": token["id"]
+ }
+ response = self.get_url(req_url, headers)
+ if not "network" in response:
+ return []
+ network = response["network"]
+ subnets = network['subnets']
+
+ # get subnets documents.
+ subnets_hash = {}
+ cidrs = []
+ subnet_ids = []
+ for subnet_id in subnets:
+ req_url = endpoint + "/v2.0/subnets/" + subnet_id
+ response = self.get_url(req_url, headers)
+ if "subnet" in response:
+ # create a hash subnets, to allow easy locating of subnets
+ subnet = response["subnet"]
+ subnets_hash[subnet["name"]] = subnet
+ cidrs.append(subnet["cidr"])
+ subnet_ids.append(subnet["id"])
+
+ network["subnets"] = subnets_hash
+ network["cidrs"] = cidrs
+ network["subnet_ids"] = subnet_ids
+
+ network["master_parent_type"] = "project"
+ network["master_parent_id"] = network["tenant_id"]
+ network["parent_type"] = "networks_folder"
+ network["parent_id"] = network["tenant_id"] + "-networks"
+ network["parent_text"] = "Networks"
+ # set the 'network' attribute for network objects to the name of network,
+ # to allow setting constraint on network when creating network clique
+ network['network'] = network["id"]
+ # get the project name
+ project = self.inv.get_by_id(self.get_env(), network["tenant_id"])
+ if project:
+ network["project"] = project["name"]
+
+ return network
diff --git a/app/discover/fetchers/api/api_fetch_networks.py b/app/discover/fetchers/api/api_fetch_networks.py
new file mode 100644
index 0000000..4b70f65
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_networks.py
@@ -0,0 +1,86 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class ApiFetchNetworks(ApiAccess):
+ def __init__(self):
+ super(ApiFetchNetworks, self).__init__()
+ self.inv = InventoryMgr()
+
+ def get(self, project_id=None):
+ # use project admin credentials, to be able to fetch all networks
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ ret = []
+ for region in self.regions:
+ ret.extend(self.get_networks(region, token))
+ return ret
+
+ def get_networks(self, region, token):
+ endpoint = self.get_region_url_nover(region, "neutron")
+ req_url = endpoint + "/v2.0/networks"
+ headers = {
+ "X-Auth-Project-Id": self.admin_project,
+ "X-Auth-Token": token["id"]
+ }
+ response = self.get_url(req_url, headers)
+ if not "networks" in response:
+ return []
+ networks = response["networks"]
+ req_url = endpoint + "/v2.0/subnets"
+ response = self.get_url(req_url, headers)
+ subnets_hash = {}
+ if "subnets" in response:
+ # create a hash subnets, to allow easy locating of subnets
+ subnets = response["subnets"]
+ for s in subnets:
+ subnets_hash[s["id"]] = s
+ for doc in networks:
+ doc["master_parent_type"] = "project"
+ project_id = doc["tenant_id"]
+ if not project_id:
+ # find project ID of admin project
+ project = self.inv.get_by_field(self.get_env(),
+ "project", "name",
+ self.admin_project,
+ get_single=True)
+ if not project:
+ self.log.error("failed to find admin project in DB")
+ project_id = project["id"]
+ doc["master_parent_id"] = project_id
+ doc["parent_type"] = "networks_folder"
+ doc["parent_id"] = project_id + "-networks"
+ doc["parent_text"] = "Networks"
+ # set the 'network' attribute for network objects to the name of network,
+ # to allow setting constraint on network when creating network clique
+ doc['network'] = doc["id"]
+ # get the project name
+ project = self.inv.get_by_id(self.get_env(), project_id)
+ if project:
+ doc["project"] = project["name"]
+ subnets_details = {}
+ cidrs = []
+ subnet_ids = []
+ for s in doc["subnets"]:
+ try:
+ subnet = subnets_hash[s]
+ cidrs.append(subnet["cidr"])
+ subnet_ids.append(subnet["id"])
+ subnets_details[subnet["name"]] = subnet
+ except KeyError:
+ pass
+
+ doc["subnets"] = subnets_details
+ doc["cidrs"] = cidrs
+ doc["subnet_ids"] = subnet_ids
+ return networks
diff --git a/app/discover/fetchers/api/api_fetch_port.py b/app/discover/fetchers/api/api_fetch_port.py
new file mode 100644
index 0000000..f8d9eeb
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_port.py
@@ -0,0 +1,60 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class ApiFetchPort(ApiAccess):
+ def __init__(self):
+ super(ApiFetchPort, self).__init__()
+ self.inv = InventoryMgr()
+
+ def get(self, project_id):
+ if not project_id:
+ self.log.info("Get method needs ID parameter")
+ return []
+ # use project admin credentials, to be able to fetch all ports
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ ret = []
+ for region in self.regions:
+ ret.append(self.get_port(region, token, project_id))
+ if ret == []:
+ self.log.info("ApiFetchPort: Port not found.")
+ return ret
+
+ def get_port(self, region, token, id):
+ endpoint = self.get_region_url_nover(region, "neutron")
+ req_url = endpoint + "/v2.0/ports/" + id
+ headers = {
+ "X-Auth-Project-Id": self.admin_project,
+ "X-Auth-Token": token["id"]
+ }
+ response = self.get_url(req_url, headers)
+ if not "port" in response:
+ return []
+
+ doc = response["port"]
+ doc["master_parent_type"] = "network"
+ doc["master_parent_id"] = doc["network_id"]
+ doc["parent_type"] = "ports_folder"
+ doc["parent_id"] = doc["network_id"] + "-ports"
+ doc["parent_text"] = "Ports"
+ # get the project name
+ net = self.inv.get_by_id(self.get_env(), doc["network_id"])
+ if net:
+ doc["name"] = doc["mac_address"]
+ else:
+ doc["name"] = doc["id"]
+ project = self.inv.get_by_id(self.get_env(), doc["tenant_id"])
+ if project:
+ doc["project"] = project["name"]
+ return doc
diff --git a/app/discover/fetchers/api/api_fetch_ports.py b/app/discover/fetchers/api/api_fetch_ports.py
new file mode 100644
index 0000000..f4c54a6
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_ports.py
@@ -0,0 +1,55 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class ApiFetchPorts(ApiAccess):
+ def __init__(self):
+ super(ApiFetchPorts, self).__init__()
+ self.inv = InventoryMgr()
+
+ def get(self, project_id):
+ # use project admin credentials, to be able to fetch all ports
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ ret = []
+ for region in self.regions:
+ ret.extend(self.get_ports_for_region(region, token))
+ return ret
+
+ def get_ports_for_region(self, region, token):
+ endpoint = self.get_region_url_nover(region, "neutron")
+ req_url = endpoint + "/v2.0/ports"
+ headers = {
+ "X-Auth-Project-Id": self.admin_project,
+ "X-Auth-Token": token["id"]
+ }
+ response = self.get_url(req_url, headers)
+ if not "ports" in response:
+ return []
+ ports = response["ports"]
+ for doc in ports:
+ doc["master_parent_type"] = "network"
+ doc["master_parent_id"] = doc["network_id"]
+ doc["parent_type"] = "ports_folder"
+ doc["parent_id"] = doc["network_id"] + "-ports"
+ doc["parent_text"] = "Ports"
+ # get the project name
+ net = self.inv.get_by_id(self.get_env(), doc["network_id"])
+ if net:
+ doc["name"] = doc["mac_address"]
+ else:
+ doc["name"] = doc["id"]
+ project = self.inv.get_by_id(self.get_env(), doc["tenant_id"])
+ if project:
+ doc["project"] = project["name"]
+ return ports
diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py
new file mode 100644
index 0000000..7dc262e
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_project_hosts.py
@@ -0,0 +1,144 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+import json
+
+from discover.fetchers.api.api_access import ApiAccess
+from discover.fetchers.db.db_access import DbAccess
+
+
+class ApiFetchProjectHosts(ApiAccess, DbAccess):
+ def __init__(self):
+ super(ApiFetchProjectHosts, self).__init__()
+
+ def get(self, project_id):
+ if project_id != self.admin_project:
+ # do not scan hosts except under project 'admin'
+ return []
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ ret = []
+ for region in self.regions:
+ ret.extend(self.get_for_region(region, token))
+ return ret
+
+ def get_for_region(self, region, token):
+ endpoint = self.get_region_url(region, "nova")
+ ret = []
+ if not token:
+ return []
+ req_url = endpoint + "/os-availability-zone/detail"
+ headers = {
+ "X-Auth-Project-Id": self.admin_project,
+ "X-Auth-Token": token["id"]
+ }
+ response = self.get_url(req_url, headers)
+ if "status" in response and int(response["status"]) != 200:
+ return []
+ az_info = response["availabilityZoneInfo"]
+ hosts = {}
+ for doc in az_info:
+ az_hosts = self.get_hosts_from_az(doc)
+ for h in az_hosts:
+ if h["name"] in hosts:
+ # merge host_type data between AZs
+ existing_entry = hosts[h["name"]]
+ for t in h["host_type"]:
+ self.add_host_type(existing_entry, t, doc['zoneName'])
+ else:
+ hosts[h["name"]] = h
+ ret.append(h)
+ # get os_id for hosts using the os-hypervisors API call
+ req_url = endpoint + "/os-hypervisors"
+ response = self.get_url(req_url, headers)
+ if "status" in response and int(response["status"]) != 200:
+ return ret
+ if "hypervisors" not in response:
+ return ret
+ for h in response["hypervisors"]:
+ hvname = h["hypervisor_hostname"]
+ if '.' in hvname and hvname not in hosts:
+ hostname = hvname[:hvname.index('.')]
+ else:
+ hostname = hvname
+ try:
+ doc = hosts[hostname]
+ except KeyError:
+ # TBD - add error output
+ continue
+ doc["os_id"] = str(h["id"])
+ self.fetch_compute_node_ip_address(doc, hvname)
+ # get more network nodes details
+ self.fetch_network_node_details(ret)
+ return ret
+
+ def get_hosts_from_az(self, az):
+ ret = []
+ for h in az["hosts"]:
+ doc = self.get_host_details(az, h)
+ ret.append(doc)
+ return ret
+
+ def get_host_details(self, az, h):
+ # for hosts we use the name
+ services = az["hosts"][h]
+ doc = {
+ "id": h,
+ "host": h,
+ "name": h,
+ "zone": az["zoneName"],
+ "parent_type": "availability_zone",
+ "parent_id": az["zoneName"],
+ "services": services,
+ "host_type": []
+ }
+ if "nova-conductor" in services:
+ s = services["nova-conductor"]
+ if s["available"] and s["active"]:
+ self.add_host_type(doc, "Controller", az['zoneName'])
+ if "nova-compute" in services:
+ s = services["nova-compute"]
+ if s["available"] and s["active"]:
+ self.add_host_type(doc, "Compute", az['zoneName'])
+ return doc
+
+ # fetch more details of network nodes from neutron.agents table
+ def fetch_network_node_details(self, docs):
+ hosts = {}
+ for doc in docs:
+ hosts[doc["host"]] = doc
+ query = """
+ SELECT DISTINCT host, host AS id, configurations
+ FROM {}.agents
+ WHERE agent_type IN ('Metadata agent', 'DHCP agent', 'L3 agent')
+ """.format(self.neutron_db)
+ results = self.get_objects_list(query, "")
+ for r in results:
+ host = hosts[r["host"]]
+ host["config"] = json.loads(r["configurations"])
+ self.add_host_type(host, "Network", '')
+
+ # fetch ip_address from nova.compute_nodes table if possible
+ def fetch_compute_node_ip_address(self, doc, h):
+ query = """
+ SELECT host_ip AS ip_address
+ FROM nova.compute_nodes
+ WHERE hypervisor_hostname = %s
+ """
+ results = self.get_objects_list_for_id(query, "", h)
+ for db_row in results:
+ doc.update(db_row)
+
+ def add_host_type(self, doc, type, zone):
+ if not type in doc["host_type"]:
+ doc["host_type"].append(type)
+ if type == 'Compute':
+ doc['zone'] = zone
+ doc['parent_id'] = zone
diff --git a/app/discover/fetchers/api/api_fetch_projects.py b/app/discover/fetchers/api/api_fetch_projects.py
new file mode 100644
index 0000000..4ef8083
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_projects.py
@@ -0,0 +1,66 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+
+
+class ApiFetchProjects(ApiAccess):
+ def __init__(self):
+ super(ApiFetchProjects, self).__init__()
+
+ def get(self, project_id):
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ if not self.regions:
+ self.log.error('No regions found')
+ return []
+ ret = []
+ for region in self.regions:
+ ret.extend(self.get_for_region(region, token))
+ projects_for_user = self.get_projects_for_api_user(region, token)
+ return [p for p in ret if p['name'] in projects_for_user] \
+ if projects_for_user else ret
+
+ def get_projects_for_api_user(self, region, token):
+ if not token:
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ endpoint = self.get_region_url_nover(region, "keystone")
+ headers = {
+ 'X-Auth-Project-Id': self.admin_project,
+ 'X-Auth-Token': token['id']
+ }
+ # get the list of projects accessible by the admin user
+ req_url = endpoint + '/v3/projects'
+ response = self.get_url(req_url, headers)
+ if not response or 'projects' not in response:
+ return None
+ response = [p['name'] for p in response['projects']]
+ return response
+
+ def get_for_region(self, region, token):
+ endpoint = self.get_region_url_nover(region, "keystone")
+ req_url = endpoint + "/v2.0/tenants"
+ headers = {
+ "X-Auth-Project-Id": self.admin_project,
+ "X-Auth-Token": token["id"]
+ }
+ response = self.get_url(req_url, headers)
+ if not isinstance(response, dict):
+ self.log.error('invalid response to /tenants request: not dict')
+ return []
+ tenants_list = response.get("tenants", [])
+ if not isinstance(tenants_list, list):
+ self.log.error('invalid response to /tenants request: '
+ 'tenants value is n ot a list')
+ return []
+ response = [t for t in tenants_list if t.get("name", "") != "services"]
+ return response
diff --git a/app/discover/fetchers/api/api_fetch_regions.py b/app/discover/fetchers/api/api_fetch_regions.py
new file mode 100644
index 0000000..dcc558f
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_regions.py
@@ -0,0 +1,51 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+
+
+class ApiFetchRegions(ApiAccess):
+ def __init__(self):
+ super(ApiFetchRegions, self).__init__()
+ self.endpoint = ApiAccess.base_url
+
+ def get(self, project_id):
+ token = self.v2_auth_pwd(self.admin_project)
+ if not token:
+ return []
+ # the returned authentication response contains the list of end points
+ # and regions
+ service_catalog = ApiAccess.auth_response.get('access', {}).get('serviceCatalog')
+ if not service_catalog:
+ return []
+ env = self.get_env()
+ ret = []
+ NULL_REGION = "No-Region"
+ for service in service_catalog:
+ for e in service["endpoints"]:
+ if "region" in e:
+ region_name = e.pop("region")
+ region_name = region_name if region_name else NULL_REGION
+ else:
+ region_name = NULL_REGION
+ if region_name in self.regions.keys():
+ region = self.regions[region_name]
+ else:
+ region = {
+ "id": region_name,
+ "name": region_name,
+ "endpoints": {}
+ }
+ ApiAccess.regions[region_name] = region
+ region["parent_type"] = "regions_folder"
+ region["parent_id"] = env + "-regions"
+ e["service_type"] = service["type"]
+ region["endpoints"][service["name"]] = e
+ ret.extend(list(ApiAccess.regions.values()))
+ return ret