Calipso initial release for OPNFV

Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88
Signed-off-by: Yaron Yogev <yaronyogev@gmail.com>

12 files changed, 892 insertions, 0 deletions

diff --git a/app/discover/fetchers/api/__init__.py b/app/discover/fetchers/api/__init__.py
new file mode 100644
index 0000000..b0637e9
--- /dev/null
+++ b/app/discover/fetchers/api/__init__.py
@@ -0,0 +1,9 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
diff --git a/app/discover/fetchers/api/api_access.py b/app/discover/fetchers/api/api_access.py
new file mode 100644
index 0000000..89eeb34
--- /dev/null
+++ b/app/discover/fetchers/api/api_access.py
@@ -0,0 +1,195 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import calendar
+import re
+import requests
+import time
+
+from discover.configuration import Configuration
+from discover.fetcher import Fetcher
+from utils.string_utils import jsonify
+
+
+class ApiAccess(Fetcher):
+    subject_token = None
+    initialized = False
+    regions = {}
+    config = None
+    api_config = None
+
+    host = ""
+    base_url = ""
+    admin_token = ""
+    tokens = {}
+    admin_endpoint = ""
+    admin_project = None
+    auth_response = None
+
+    alternative_services = {
+        "neutron": ["quantum"]
+    }
+
+    # identitity API v2 version with admin token
+    def __init__(self):
+        super(ApiAccess, self).__init__()
+        if ApiAccess.initialized:
+            return
+        ApiAccess.config = Configuration()
+        ApiAccess.api_config = ApiAccess.config.get("OpenStack")
+        host = ApiAccess.api_config["host"]
+        ApiAccess.host = host
+        port = ApiAccess.api_config["port"]
+        if not (host and port):
+            raise ValueError('Missing definition of host or port ' +
+                             'for OpenStack API access')
+        ApiAccess.base_url = "http://" + host + ":" + port
+        ApiAccess.admin_token = ApiAccess.api_config["admin_token"]
+        ApiAccess.admin_project = ApiAccess.api_config["admin_project"] \
+            if "admin_project" in ApiAccess.api_config \
+            else 'admin'
+        ApiAccess.admin_endpoint = "http://" + host + ":" + "35357"
+
+        token = self.v2_auth_pwd(ApiAccess.admin_project)
+        if not token:
+            raise ValueError("Authentication failed. Failed to obtain token")
+        else:
+            self.subject_token = token
+
+    @staticmethod
+    def parse_time(time_str):
+        try:
+            time_struct = time.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ")
+        except ValueError:
+            try:
+                time_struct = time.strptime(time_str,
+                                            "%Y-%m-%dT%H:%M:%S.%fZ")
+            except ValueError:
+                return None
+        return time_struct
+
+    # try to use existing token, if it did not expire
+    def get_existing_token(self, project_id):
+        try:
+            token_details = ApiAccess.tokens[project_id]
+        except KeyError:
+            return None
+        token_expiry = token_details["expires"]
+        token_expiry_time_struct = self.parse_time(token_expiry)
+        if not token_expiry_time_struct:
+            return None
+        token_expiry_time = token_details["token_expiry_time"]
+        now = time.time()
+        if now > token_expiry_time:
+            # token has expired
+            ApiAccess.tokens.pop(project_id)
+            return None
+        return token_details
+
+    def v2_auth(self, project_id, headers, post_body):
+        subject_token = self.get_existing_token(project_id)
+        if subject_token:
+            return subject_token
+        req_url = ApiAccess.base_url + "/v2.0/tokens"
+        response = requests.post(req_url, json=post_body, headers=headers)
+        ApiAccess.auth_response = response.json()
+        if 'error' in self.auth_response:
+            e = self.auth_response['error']
+            self.log.error(str(e['code']) + ' ' + e['title'] + ': ' +
+                           e['message'] + ", URL: " + req_url)
+            return None
+        try:
+            token_details = ApiAccess.auth_response["access"]["token"]
+        except KeyError:
+            # assume authentication failed
+            return None
+        token_expiry = token_details["expires"]
+        token_expiry_time_struct = self.parse_time(token_expiry)
+        if not token_expiry_time_struct:
+            return None
+        token_expiry_time = calendar.timegm(token_expiry_time_struct)
+        token_details["token_expiry_time"] = token_expiry_time
+        ApiAccess.tokens[project_id] = token_details
+        return token_details
+
+    def v2_auth_pwd(self, project):
+        user = ApiAccess.api_config["user"]
+        pwd = ApiAccess.api_config["pwd"]
+        post_body = {
+            "auth": {
+                "passwordCredentials": {
+                    "username": user,
+                    "password": pwd
+                }
+            }
+        }
+        if project is not None:
+            post_body["auth"]["tenantName"] = project
+            project_id = project
+        else:
+            project_id = ""
+        headers = {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json; charset=UTF-8'
+        }
+        return self.v2_auth(project_id, headers, post_body)
+
+    def get_rel_url(self, relative_url, headers):
+        req_url = ApiAccess.base_url + relative_url
+        return self.get_url(req_url, headers)
+
+    def get_url(self, req_url, headers):
+        response = requests.get(req_url, headers=headers)
+        if response.status_code != requests.codes.ok:
+            # some error happened
+            if "reason" in response:
+                msg = ", reason: {}".format(response.reason)
+            else:
+                msg = ", response: {}".format(response.text)
+            self.log.error("req_url: {} {}".format(req_url, msg))
+            return response
+        ret = response.json()
+        return ret
+
+    def get_region_url(self, region_name, service):
+        if region_name not in self.regions:
+            return None
+        region = self.regions[region_name]
+        s = self.get_service_region_endpoints(region, service)
+        if not s:
+            return None
+        orig_url = s["adminURL"]
+        # replace host name with the host found in config
+        url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url)
+        return url
+
+    # like get_region_url(), but remove everything starting from the "/v2"
+    def get_region_url_nover(self, region, service):
+        full_url = self.get_region_url(region, service)
+        if not full_url:
+            self.log.error("could not find region URL for region: " + region)
+            exit()
+        url = re.sub(r":([0-9]+)/v[2-9].*", r":\1", full_url)
+        return url
+
+    def get_catalog(self, pretty):
+        return jsonify(self.regions, pretty)
+
+    # find the endpoints for a given service name,
+    # considering also alternative service names
+    def get_service_region_endpoints(self, region, service):
+        alternatives = [service]
+        endpoints = region["endpoints"]
+        if service in self.alternative_services:
+            alternatives.extend(self.alternative_services[service])
+        for sname in alternatives:
+            if sname in endpoints:
+                return endpoints[sname]
+        return None
+
diff --git a/app/discover/fetchers/api/api_fetch_availability_zones.py b/app/discover/fetchers/api/api_fetch_availability_zones.py
new file mode 100644
index 0000000..196893b
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_availability_zones.py
@@ -0,0 +1,56 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+
+
+class ApiFetchAvailabilityZones(ApiAccess):
+    def __init__(self):
+        super(ApiFetchAvailabilityZones, self).__init__()
+
+    def get(self, project_id):
+        token = self.v2_auth_pwd(project_id)
+        if not token:
+            return []
+        ret = []
+        for region in self.regions:
+            ret.extend(self.get_for_region(project_id, region, token))
+        return ret
+
+    def get_for_region(self, project, region, token):
+        # we use os-availability-zone/detail rather than os-availability-zone,
+        # because the later does not inclde the "internal" zone in the results
+        endpoint = self.get_region_url_nover(region, "nova")
+        req_url = endpoint + "/v2/" + token["tenant"]["id"] + \
+                  "/os-availability-zone/detail"
+        headers = {
+            "X-Auth-Project-Id": project,
+            "X-Auth-Token": token["id"]
+        }
+        response = self.get_url(req_url, headers)
+        if "status" in response and int(response["status"]) != 200:
+            return []
+        ret = []
+        if "availabilityZoneInfo" not in response:
+            return []
+        azs = response["availabilityZoneInfo"]
+        if not azs:
+            return []
+        for doc in azs:
+            doc["id"] = doc["zoneName"]
+            doc["name"] = doc.pop("zoneName")
+            doc["master_parent_type"] = "region"
+            doc["master_parent_id"] = region
+            doc["parent_type"] = "availability_zones_folder"
+            doc["parent_id"] = region + "-availability_zones"
+            doc["parent_text"] = "Availability Zones"
+            doc["available"] = doc["zoneState"]["available"]
+            doc.pop("zoneState")
+            ret.append(doc)
+        return ret
diff --git a/app/discover/fetchers/api/api_fetch_end_points.py b/app/discover/fetchers/api/api_fetch_end_points.py
new file mode 100644
index 0000000..9471c7e
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_end_points.py
@@ -0,0 +1,35 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# fetch the end points for a given project (tenant)
+# return list of regions, to allow further recursive scanning
+
+from discover.fetchers.api.api_access import ApiAccess
+
+
+class ApiFetchEndPoints(ApiAccess):
+
+    def get(self, project_id):
+        if project_id != "admin":
+            return []  # XXX currently having problems authenticating to other tenants
+        self.v2_auth_pwd(project_id)
+
+        environment = ApiAccess.config.get_env_name()
+        regions = []
+        services = ApiAccess.auth_response['access']['serviceCatalog']
+        endpoints = []
+        for s in services:
+            if s["type"] != "identity":
+                continue
+            e = s["endpoints"][0]
+            e["environment"] = environment
+            e["project"] = project_id
+            e["type"] = "endpoint"
+            endpoints.append(e)
+        return endpoints
diff --git a/app/discover/fetchers/api/api_fetch_host_instances.py b/app/discover/fetchers/api/api_fetch_host_instances.py
new file mode 100644
index 0000000..56cffda
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_host_instances.py
@@ -0,0 +1,59 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from discover.fetchers.db.db_access import DbAccess
+from discover.fetchers.db.db_fetch_instances import DbFetchInstances
+from utils.inventory_mgr import InventoryMgr
+from utils.singleton import Singleton
+
+
+class ApiFetchHostInstances(ApiAccess, DbAccess, metaclass=Singleton):
+    def __init__(self):
+        super(ApiFetchHostInstances, self).__init__()
+        self.inv = InventoryMgr()
+        self.endpoint = ApiAccess.base_url.replace(":5000", ":8774")
+        self.projects = None
+        self.db_fetcher = DbFetchInstances()
+
+    def get_projects(self):
+        if not self.projects:
+            projects_list = self.inv.get(self.get_env(), "project", None)
+            self.projects = [p["name"] for p in projects_list]
+
+    def get(self, id):
+        self.get_projects()
+        host_id = id[:id.rindex("-")]
+        host = self.inv.get_by_id(self.get_env(), host_id)
+        if not host or "Compute" not in host.get("host_type", ""):
+            return []
+        instances_found = self.get_instances_from_api(host_id)
+        self.db_fetcher.get_instance_data(instances_found)
+        return instances_found
+
+    def get_instances_from_api(self, host_name):
+        token = self.v2_auth_pwd(self.admin_project)
+        if not token:
+            return []
+        tenant_id = token["tenant"]["id"]
+        req_url = self.endpoint + "/v2/" + tenant_id + \
+                  "/os-hypervisors/" + host_name + "/servers"
+        response = self.get_url(req_url, {"X-Auth-Token": token["id"]})
+        ret = []
+        if not "hypervisors" in response:
+            return []
+        if not "servers" in response["hypervisors"][0]:
+            return []
+        for doc in response["hypervisors"][0]["servers"]:
+            doc["id"] = doc["uuid"]
+            doc["host"] = host_name
+            doc["local_name"] = doc.pop("name")
+            ret.append(doc)
+        self.log.info("found %s instances for host: %s", str(len(ret)), host_name)
+        return ret
diff --git a/app/discover/fetchers/api/api_fetch_network.py b/app/discover/fetchers/api/api_fetch_network.py
new file mode 100644
index 0000000..889b8a5
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_network.py
@@ -0,0 +1,76 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class ApiFetchNetwork(ApiAccess):
+    def __init__(self):
+        super(ApiFetchNetwork, self).__init__()
+        self.inv = InventoryMgr()
+
+    def get(self, project_id):
+        # use project admin credentials, to be able to fetch all networks
+        token = self.v2_auth_pwd(self.admin_project)
+        if not token:
+            return []
+        ret = []
+        for region in self.regions:
+            # TODO: refactor legacy code (Unresolved reference - self.get_for_region)
+            ret.extend(self.get_for_region(region, token, project_id))
+        return ret
+
+    def get_network(self, region, token, subnet_id):
+        endpoint = self.get_region_url_nover(region, "neutron")
+
+        # get target network network document
+        req_url = endpoint + "/v2.0/networks/" + subnet_id
+        headers = {
+            "X-Auth-Project-Id": self.admin_project,
+            "X-Auth-Token": token["id"]
+        }
+        response = self.get_url(req_url, headers)
+        if not "network" in response:
+            return []
+        network = response["network"]
+        subnets = network['subnets']
+
+        # get subnets documents.
+        subnets_hash = {}
+        cidrs = []
+        subnet_ids = []
+        for subnet_id in subnets:
+            req_url = endpoint + "/v2.0/subnets/" + subnet_id
+            response = self.get_url(req_url, headers)
+            if "subnet" in response:
+                # create a hash subnets, to allow easy locating of subnets
+                subnet = response["subnet"]
+                subnets_hash[subnet["name"]] = subnet
+                cidrs.append(subnet["cidr"])
+                subnet_ids.append(subnet["id"])
+
+        network["subnets"] = subnets_hash
+        network["cidrs"] = cidrs
+        network["subnet_ids"] = subnet_ids
+
+        network["master_parent_type"] = "project"
+        network["master_parent_id"] = network["tenant_id"]
+        network["parent_type"] = "networks_folder"
+        network["parent_id"] = network["tenant_id"] + "-networks"
+        network["parent_text"] = "Networks"
+        # set the 'network' attribute for network objects to the name of network,
+        # to allow setting constraint on network when creating network clique
+        network['network'] = network["id"]
+        # get the project name
+        project = self.inv.get_by_id(self.get_env(), network["tenant_id"])
+        if project:
+            network["project"] = project["name"]
+
+        return network
diff --git a/app/discover/fetchers/api/api_fetch_networks.py b/app/discover/fetchers/api/api_fetch_networks.py
new file mode 100644
index 0000000..4b70f65
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_networks.py
@@ -0,0 +1,86 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class ApiFetchNetworks(ApiAccess):
+    def __init__(self):
+        super(ApiFetchNetworks, self).__init__()
+        self.inv = InventoryMgr()
+
+    def get(self, project_id=None):
+        # use project admin credentials, to be able to fetch all networks
+        token = self.v2_auth_pwd(self.admin_project)
+        if not token:
+            return []
+        ret = []
+        for region in self.regions:
+            ret.extend(self.get_networks(region, token))
+        return ret
+
+    def get_networks(self, region, token):
+        endpoint = self.get_region_url_nover(region, "neutron")
+        req_url = endpoint + "/v2.0/networks"
+        headers = {
+            "X-Auth-Project-Id": self.admin_project,
+            "X-Auth-Token": token["id"]
+        }
+        response = self.get_url(req_url, headers)
+        if not "networks" in response:
+            return []
+        networks = response["networks"]
+        req_url = endpoint + "/v2.0/subnets"
+        response = self.get_url(req_url, headers)
+        subnets_hash = {}
+        if "subnets" in response:
+            # create a hash subnets, to allow easy locating of subnets
+            subnets = response["subnets"]
+            for s in subnets:
+                subnets_hash[s["id"]] = s
+        for doc in networks:
+            doc["master_parent_type"] = "project"
+            project_id = doc["tenant_id"]
+            if not project_id:
+                # find project ID of admin project
+                project = self.inv.get_by_field(self.get_env(),
+                                                "project", "name",
+                                                self.admin_project,
+                                                get_single=True)
+                if not project:
+                    self.log.error("failed to find admin project in DB")
+                project_id = project["id"]
+            doc["master_parent_id"] = project_id
+            doc["parent_type"] = "networks_folder"
+            doc["parent_id"] = project_id + "-networks"
+            doc["parent_text"] = "Networks"
+            # set the 'network' attribute for network objects to the name of network,
+            # to allow setting constraint on network when creating network clique
+            doc['network'] = doc["id"]
+            # get the project name
+            project = self.inv.get_by_id(self.get_env(), project_id)
+            if project:
+                doc["project"] = project["name"]
+            subnets_details = {}
+            cidrs = []
+            subnet_ids = []
+            for s in doc["subnets"]:
+                try:
+                    subnet = subnets_hash[s]
+                    cidrs.append(subnet["cidr"])
+                    subnet_ids.append(subnet["id"])
+                    subnets_details[subnet["name"]] = subnet
+                except KeyError:
+                    pass
+
+            doc["subnets"] = subnets_details
+            doc["cidrs"] = cidrs
+            doc["subnet_ids"] = subnet_ids
+        return networks
diff --git a/app/discover/fetchers/api/api_fetch_port.py b/app/discover/fetchers/api/api_fetch_port.py
new file mode 100644
index 0000000..f8d9eeb
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_port.py
@@ -0,0 +1,60 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class ApiFetchPort(ApiAccess):
+    def __init__(self):
+        super(ApiFetchPort, self).__init__()
+        self.inv = InventoryMgr()
+
+    def get(self, project_id):
+        if not project_id:
+            self.log.info("Get method needs ID parameter")
+            return []
+        # use project admin credentials, to be able to fetch all ports
+        token = self.v2_auth_pwd(self.admin_project)
+        if not token:
+            return []
+        ret = []
+        for region in self.regions:
+            ret.append(self.get_port(region, token, project_id))
+        if ret == []:
+            self.log.info("ApiFetchPort: Port not found.")
+        return ret
+
+    def get_port(self, region, token, id):
+        endpoint = self.get_region_url_nover(region, "neutron")
+        req_url = endpoint + "/v2.0/ports/" + id
+        headers = {
+            "X-Auth-Project-Id": self.admin_project,
+            "X-Auth-Token": token["id"]
+        }
+        response = self.get_url(req_url, headers)
+        if not "port" in response:
+            return []
+
+        doc = response["port"]
+        doc["master_parent_type"] = "network"
+        doc["master_parent_id"] = doc["network_id"]
+        doc["parent_type"] = "ports_folder"
+        doc["parent_id"] = doc["network_id"] + "-ports"
+        doc["parent_text"] = "Ports"
+        # get the project name
+        net = self.inv.get_by_id(self.get_env(), doc["network_id"])
+        if net:
+            doc["name"] = doc["mac_address"]
+        else:
+            doc["name"] = doc["id"]
+        project = self.inv.get_by_id(self.get_env(), doc["tenant_id"])
+        if project:
+            doc["project"] = project["name"]
+        return doc
diff --git a/app/discover/fetchers/api/api_fetch_ports.py b/app/discover/fetchers/api/api_fetch_ports.py
new file mode 100644
index 0000000..f4c54a6
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_ports.py
@@ -0,0 +1,55 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class ApiFetchPorts(ApiAccess):
+    def __init__(self):
+        super(ApiFetchPorts, self).__init__()
+        self.inv = InventoryMgr()
+
+    def get(self, project_id):
+        # use project admin credentials, to be able to fetch all ports
+        token = self.v2_auth_pwd(self.admin_project)
+        if not token:
+            return []
+        ret = []
+        for region in self.regions:
+            ret.extend(self.get_ports_for_region(region, token))
+        return ret
+
+    def get_ports_for_region(self, region, token):
+        endpoint = self.get_region_url_nover(region, "neutron")
+        req_url = endpoint + "/v2.0/ports"
+        headers = {
+            "X-Auth-Project-Id": self.admin_project,
+            "X-Auth-Token": token["id"]
+        }
+        response = self.get_url(req_url, headers)
+        if not "ports" in response:
+            return []
+        ports = response["ports"]
+        for doc in ports:
+            doc["master_parent_type"] = "network"
+            doc["master_parent_id"] = doc["network_id"]
+            doc["parent_type"] = "ports_folder"
+            doc["parent_id"] = doc["network_id"] + "-ports"
+            doc["parent_text"] = "Ports"
+            # get the project name
+            net = self.inv.get_by_id(self.get_env(), doc["network_id"])
+            if net:
+                doc["name"] = doc["mac_address"]
+            else:
+                doc["name"] = doc["id"]
+            project = self.inv.get_by_id(self.get_env(), doc["tenant_id"])
+            if project:
+                doc["project"] = project["name"]
+        return ports
diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py
new file mode 100644
index 0000000..7dc262e
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_project_hosts.py
@@ -0,0 +1,144 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import json
+
+from discover.fetchers.api.api_access import ApiAccess
+from discover.fetchers.db.db_access import DbAccess
+
+
+class ApiFetchProjectHosts(ApiAccess, DbAccess):
+    def __init__(self):
+        super(ApiFetchProjectHosts, self).__init__()
+
+    def get(self, project_id):
+        if project_id != self.admin_project:
+            # do not scan hosts except under project 'admin'
+            return []
+        token = self.v2_auth_pwd(self.admin_project)
+        if not token:
+            return []
+        ret = []
+        for region in self.regions:
+            ret.extend(self.get_for_region(region, token))
+        return ret
+
+    def get_for_region(self, region, token):
+        endpoint = self.get_region_url(region, "nova")
+        ret = []
+        if not token:
+            return []
+        req_url = endpoint + "/os-availability-zone/detail"
+        headers = {
+            "X-Auth-Project-Id": self.admin_project,
+            "X-Auth-Token": token["id"]
+        }
+        response = self.get_url(req_url, headers)
+        if "status" in response and int(response["status"]) != 200:
+            return []
+        az_info = response["availabilityZoneInfo"]
+        hosts = {}
+        for doc in az_info:
+            az_hosts = self.get_hosts_from_az(doc)
+            for h in az_hosts:
+                if h["name"] in hosts:
+                    # merge host_type data between AZs
+                    existing_entry = hosts[h["name"]]
+                    for t in h["host_type"]:
+                        self.add_host_type(existing_entry, t, doc['zoneName'])
+                else:
+                    hosts[h["name"]] = h
+                    ret.append(h)
+        # get os_id for hosts using the os-hypervisors API call
+        req_url = endpoint + "/os-hypervisors"
+        response = self.get_url(req_url, headers)
+        if "status" in response and int(response["status"]) != 200:
+            return ret
+        if "hypervisors" not in response:
+            return ret
+        for h in response["hypervisors"]:
+            hvname = h["hypervisor_hostname"]
+            if '.' in hvname and hvname not in hosts:
+                hostname = hvname[:hvname.index('.')]
+            else:
+                hostname = hvname
+            try:
+                doc = hosts[hostname]
+            except KeyError:
+                # TBD - add error output
+                continue
+            doc["os_id"] = str(h["id"])
+            self.fetch_compute_node_ip_address(doc, hvname)
+        # get more network nodes details
+        self.fetch_network_node_details(ret)
+        return ret
+
+    def get_hosts_from_az(self, az):
+        ret = []
+        for h in az["hosts"]:
+            doc = self.get_host_details(az, h)
+            ret.append(doc)
+        return ret
+
+    def get_host_details(self, az, h):
+        # for hosts we use the name
+        services = az["hosts"][h]
+        doc = {
+            "id": h,
+            "host": h,
+            "name": h,
+            "zone": az["zoneName"],
+            "parent_type": "availability_zone",
+            "parent_id": az["zoneName"],
+            "services": services,
+            "host_type": []
+        }
+        if "nova-conductor" in services:
+            s = services["nova-conductor"]
+            if s["available"] and s["active"]:
+                self.add_host_type(doc, "Controller", az['zoneName'])
+        if "nova-compute" in services:
+            s = services["nova-compute"]
+            if s["available"] and s["active"]:
+                self.add_host_type(doc, "Compute", az['zoneName'])
+        return doc
+
+    # fetch more details of network nodes from neutron.agents table
+    def fetch_network_node_details(self, docs):
+        hosts = {}
+        for doc in docs:
+            hosts[doc["host"]] = doc
+        query = """
+          SELECT DISTINCT host, host AS id, configurations
+          FROM {}.agents
+          WHERE agent_type IN ('Metadata agent', 'DHCP agent', 'L3 agent')
+        """.format(self.neutron_db)
+        results = self.get_objects_list(query, "")
+        for r in results:
+            host = hosts[r["host"]]
+            host["config"] = json.loads(r["configurations"])
+            self.add_host_type(host, "Network", '')
+
+    # fetch ip_address from nova.compute_nodes table if possible
+    def fetch_compute_node_ip_address(self, doc, h):
+        query = """
+      SELECT host_ip AS ip_address
+      FROM nova.compute_nodes
+      WHERE hypervisor_hostname = %s
+    """
+        results = self.get_objects_list_for_id(query, "", h)
+        for db_row in results:
+            doc.update(db_row)
+
+    def add_host_type(self, doc, type, zone):
+        if not type in doc["host_type"]:
+            doc["host_type"].append(type)
+            if type == 'Compute':
+                doc['zone'] = zone
+                doc['parent_id'] = zone
diff --git a/app/discover/fetchers/api/api_fetch_projects.py b/app/discover/fetchers/api/api_fetch_projects.py
new file mode 100644
index 0000000..4ef8083
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_projects.py
@@ -0,0 +1,66 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+
+
+class ApiFetchProjects(ApiAccess):
+    def __init__(self):
+        super(ApiFetchProjects, self).__init__()
+
+    def get(self, project_id):
+        token = self.v2_auth_pwd(self.admin_project)
+        if not token:
+            return []
+        if not self.regions:
+            self.log.error('No regions found')
+            return []
+        ret = []
+        for region in self.regions:
+            ret.extend(self.get_for_region(region, token))
+        projects_for_user = self.get_projects_for_api_user(region, token)
+        return [p for p in ret if p['name'] in projects_for_user] \
+            if projects_for_user else ret
+
+    def get_projects_for_api_user(self, region, token):
+        if not token:
+            token = self.v2_auth_pwd(self.admin_project)
+            if not token:
+                return []
+        endpoint = self.get_region_url_nover(region, "keystone")
+        headers = {
+            'X-Auth-Project-Id': self.admin_project,
+            'X-Auth-Token': token['id']
+        }
+        # get the list of projects accessible by the admin user
+        req_url = endpoint + '/v3/projects'
+        response = self.get_url(req_url, headers)
+        if not response or 'projects' not in response:
+            return None
+        response = [p['name'] for p in response['projects']]
+        return response
+
+    def get_for_region(self, region, token):
+        endpoint = self.get_region_url_nover(region, "keystone")
+        req_url = endpoint + "/v2.0/tenants"
+        headers = {
+            "X-Auth-Project-Id": self.admin_project,
+            "X-Auth-Token": token["id"]
+        }
+        response = self.get_url(req_url, headers)
+        if not isinstance(response, dict):
+            self.log.error('invalid response to /tenants request: not dict')
+            return []
+        tenants_list = response.get("tenants", [])
+        if not isinstance(tenants_list, list):
+            self.log.error('invalid response to /tenants request: '
+                           'tenants value is n ot a list')
+            return []
+        response = [t for t in tenants_list if t.get("name", "") != "services"]
+        return response
diff --git a/app/discover/fetchers/api/api_fetch_regions.py b/app/discover/fetchers/api/api_fetch_regions.py
new file mode 100644
index 0000000..dcc558f
--- /dev/null
+++ b/app/discover/fetchers/api/api_fetch_regions.py
@@ -0,0 +1,51 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.api.api_access import ApiAccess
+
+
+class ApiFetchRegions(ApiAccess):
+    def __init__(self):
+        super(ApiFetchRegions, self).__init__()
+        self.endpoint = ApiAccess.base_url
+
+    def get(self, project_id):
+        token = self.v2_auth_pwd(self.admin_project)
+        if not token:
+            return []
+        # the returned authentication response contains the list of end points
+        # and regions
+        service_catalog = ApiAccess.auth_response.get('access', {}).get('serviceCatalog')
+        if not service_catalog:
+            return []
+        env = self.get_env()
+        ret = []
+        NULL_REGION = "No-Region"
+        for service in service_catalog:
+            for e in service["endpoints"]:
+                if "region" in e:
+                    region_name = e.pop("region")
+                    region_name = region_name if region_name else NULL_REGION
+                else:
+                    region_name = NULL_REGION
+                if region_name in self.regions.keys():
+                    region = self.regions[region_name]
+                else:
+                    region = {
+                        "id": region_name,
+                        "name": region_name,
+                        "endpoints": {}
+                    }
+                    ApiAccess.regions[region_name] = region
+                region["parent_type"] = "regions_folder"
+                region["parent_id"] = env + "-regions"
+                e["service_type"] = service["type"]
+                region["endpoints"][service["name"]] = e
+        ret.extend(list(ApiAccess.regions.values()))
+        return ret