DE811 fix fail on 2nd scan in scan manager

- keep auth_response per project
- also deleted unused ApiFetchEndPoints

Change-Id: I7e349431239a4710992eb2824881dc0f1be1704a
Signed-off-by: yayogev <yaronyogev@gmail.com>

3 files changed, 255 insertions, 281 deletions

diff --git a/app/discover/fetchers/api/api_access.py b/app/discover/fetchers/api/api_access.py
index 89eeb34..3250378 100644
--- a/app/discover/fetchers/api/api_access.py
+++ b/app/discover/fetchers/api/api_access.py
@@ -1,195 +1,202 @@
-###############################################################################
-# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
-# and others                                                                  #
-#                                                                             #
-# All rights reserved. This program and the accompanying materials            #
-# are made available under the terms of the Apache License, Version 2.0       #
-# which accompanies this distribution, and is available at                    #
-# http://www.apache.org/licenses/LICENSE-2.0                                  #
-###############################################################################
-import calendar
-import re
-import requests
-import time
-
-from discover.configuration import Configuration
-from discover.fetcher import Fetcher
-from utils.string_utils import jsonify
-
-
-class ApiAccess(Fetcher):
-    subject_token = None
-    initialized = False
-    regions = {}
-    config = None
-    api_config = None
-
-    host = ""
-    base_url = ""
-    admin_token = ""
-    tokens = {}
-    admin_endpoint = ""
-    admin_project = None
-    auth_response = None
-
-    alternative_services = {
-        "neutron": ["quantum"]
-    }
-
-    # identitity API v2 version with admin token
-    def __init__(self):
-        super(ApiAccess, self).__init__()
-        if ApiAccess.initialized:
-            return
-        ApiAccess.config = Configuration()
-        ApiAccess.api_config = ApiAccess.config.get("OpenStack")
-        host = ApiAccess.api_config["host"]
-        ApiAccess.host = host
-        port = ApiAccess.api_config["port"]
-        if not (host and port):
-            raise ValueError('Missing definition of host or port ' +
-                             'for OpenStack API access')
-        ApiAccess.base_url = "http://" + host + ":" + port
-        ApiAccess.admin_token = ApiAccess.api_config["admin_token"]
-        ApiAccess.admin_project = ApiAccess.api_config["admin_project"] \
-            if "admin_project" in ApiAccess.api_config \
-            else 'admin'
-        ApiAccess.admin_endpoint = "http://" + host + ":" + "35357"
-
-        token = self.v2_auth_pwd(ApiAccess.admin_project)
-        if not token:
-            raise ValueError("Authentication failed. Failed to obtain token")
-        else:
-            self.subject_token = token
-
-    @staticmethod
-    def parse_time(time_str):
-        try:
-            time_struct = time.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ")
-        except ValueError:
-            try:
-                time_struct = time.strptime(time_str,
-                                            "%Y-%m-%dT%H:%M:%S.%fZ")
-            except ValueError:
-                return None
-        return time_struct
-
-    # try to use existing token, if it did not expire
-    def get_existing_token(self, project_id):
-        try:
-            token_details = ApiAccess.tokens[project_id]
-        except KeyError:
-            return None
-        token_expiry = token_details["expires"]
-        token_expiry_time_struct = self.parse_time(token_expiry)
-        if not token_expiry_time_struct:
-            return None
-        token_expiry_time = token_details["token_expiry_time"]
-        now = time.time()
-        if now > token_expiry_time:
-            # token has expired
-            ApiAccess.tokens.pop(project_id)
-            return None
-        return token_details
-
-    def v2_auth(self, project_id, headers, post_body):
-        subject_token = self.get_existing_token(project_id)
-        if subject_token:
-            return subject_token
-        req_url = ApiAccess.base_url + "/v2.0/tokens"
-        response = requests.post(req_url, json=post_body, headers=headers)
-        ApiAccess.auth_response = response.json()
-        if 'error' in self.auth_response:
-            e = self.auth_response['error']
-            self.log.error(str(e['code']) + ' ' + e['title'] + ': ' +
-                           e['message'] + ", URL: " + req_url)
-            return None
-        try:
-            token_details = ApiAccess.auth_response["access"]["token"]
-        except KeyError:
-            # assume authentication failed
-            return None
-        token_expiry = token_details["expires"]
-        token_expiry_time_struct = self.parse_time(token_expiry)
-        if not token_expiry_time_struct:
-            return None
-        token_expiry_time = calendar.timegm(token_expiry_time_struct)
-        token_details["token_expiry_time"] = token_expiry_time
-        ApiAccess.tokens[project_id] = token_details
-        return token_details
-
-    def v2_auth_pwd(self, project):
-        user = ApiAccess.api_config["user"]
-        pwd = ApiAccess.api_config["pwd"]
-        post_body = {
-            "auth": {
-                "passwordCredentials": {
-                    "username": user,
-                    "password": pwd
-                }
-            }
-        }
-        if project is not None:
-            post_body["auth"]["tenantName"] = project
-            project_id = project
-        else:
-            project_id = ""
-        headers = {
-            'Accept': 'application/json',
-            'Content-Type': 'application/json; charset=UTF-8'
-        }
-        return self.v2_auth(project_id, headers, post_body)
-
-    def get_rel_url(self, relative_url, headers):
-        req_url = ApiAccess.base_url + relative_url
-        return self.get_url(req_url, headers)
-
-    def get_url(self, req_url, headers):
-        response = requests.get(req_url, headers=headers)
-        if response.status_code != requests.codes.ok:
-            # some error happened
-            if "reason" in response:
-                msg = ", reason: {}".format(response.reason)
-            else:
-                msg = ", response: {}".format(response.text)
-            self.log.error("req_url: {} {}".format(req_url, msg))
-            return response
-        ret = response.json()
-        return ret
-
-    def get_region_url(self, region_name, service):
-        if region_name not in self.regions:
-            return None
-        region = self.regions[region_name]
-        s = self.get_service_region_endpoints(region, service)
-        if not s:
-            return None
-        orig_url = s["adminURL"]
-        # replace host name with the host found in config
-        url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url)
-        return url
-
-    # like get_region_url(), but remove everything starting from the "/v2"
-    def get_region_url_nover(self, region, service):
-        full_url = self.get_region_url(region, service)
-        if not full_url:
-            self.log.error("could not find region URL for region: " + region)
-            exit()
-        url = re.sub(r":([0-9]+)/v[2-9].*", r":\1", full_url)
-        return url
-
-    def get_catalog(self, pretty):
-        return jsonify(self.regions, pretty)
-
-    # find the endpoints for a given service name,
-    # considering also alternative service names
-    def get_service_region_endpoints(self, region, service):
-        alternatives = [service]
-        endpoints = region["endpoints"]
-        if service in self.alternative_services:
-            alternatives.extend(self.alternative_services[service])
-        for sname in alternatives:
-            if sname in endpoints:
-                return endpoints[sname]
-        return None
-
+###############################################################################

+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #

+# and others                                                                  #

+#                                                                             #

+# All rights reserved. This program and the accompanying materials            #

+# are made available under the terms of the Apache License, Version 2.0       #

+# which accompanies this distribution, and is available at                    #

+# http://www.apache.org/licenses/LICENSE-2.0                                  #

+###############################################################################

+import calendar

+import re

+import requests

+import time

+

+from discover.configuration import Configuration

+from discover.fetcher import Fetcher

+from utils.string_utils import jsonify

+

+

+class ApiAccess(Fetcher):

+    subject_token = None

+    initialized = False

+    regions = {}

+    config = None

+    api_config = None

+

+    host = ""

+    base_url = ""

+    admin_token = ""

+    tokens = {}

+    admin_endpoint = ""

+    admin_project = None

+    auth_response = {}

+

+    alternative_services = {

+        "neutron": ["quantum"]

+    }

+

+    # identitity API v2 version with admin token

+    def __init__(self):

+        super(ApiAccess, self).__init__()

+        if ApiAccess.initialized:

+            return

+        ApiAccess.config = Configuration()

+        ApiAccess.api_config = ApiAccess.config.get("OpenStack")

+        host = ApiAccess.api_config["host"]

+        ApiAccess.host = host

+        port = ApiAccess.api_config["port"]

+        if not (host and port):

+            raise ValueError('Missing definition of host or port ' +

+                             'for OpenStack API access')

+        ApiAccess.base_url = "http://" + host + ":" + port

+        ApiAccess.admin_token = ApiAccess.api_config["admin_token"]

+        ApiAccess.admin_project = ApiAccess.api_config["admin_project"] \

+            if "admin_project" in ApiAccess.api_config \

+            else 'admin'

+        ApiAccess.admin_endpoint = "http://" + host + ":" + "35357"

+

+        token = self.v2_auth_pwd(ApiAccess.admin_project)

+        if not token:

+            raise ValueError("Authentication failed. Failed to obtain token")

+        else:

+            self.subject_token = token

+

+    @staticmethod

+    def parse_time(time_str):

+        try:

+            time_struct = time.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ")

+        except ValueError:

+            try:

+                time_struct = time.strptime(time_str,

+                                            "%Y-%m-%dT%H:%M:%S.%fZ")

+            except ValueError:

+                return None

+        return time_struct

+

+    # try to use existing token, if it did not expire

+    def get_existing_token(self, project_id):

+        try:

+            token_details = ApiAccess.tokens[project_id]

+        except KeyError:

+            return None

+        token_expiry = token_details["expires"]

+        token_expiry_time_struct = self.parse_time(token_expiry)

+        if not token_expiry_time_struct:

+            return None

+        token_expiry_time = token_details["token_expiry_time"]

+        now = time.time()

+        if now > token_expiry_time:

+            # token has expired

+            ApiAccess.tokens.pop(project_id)

+            return None

+        return token_details

+

+    def v2_auth(self, project_id, headers, post_body):

+        subject_token = self.get_existing_token(project_id)

+        if subject_token:

+            return subject_token

+        req_url = ApiAccess.base_url + "/v2.0/tokens"

+        response = requests.post(req_url, json=post_body, headers=headers)

+        response = response.json()

+        ApiAccess.auth_response[project_id] = response

+        if 'error' in response:

+            e = response['error']

+            self.log.error(str(e['code']) + ' ' + e['title'] + ': ' +

+                           e['message'] + ", URL: " + req_url)

+            return None

+        try:

+            token_details = response["access"]["token"]

+        except KeyError:

+            # assume authentication failed

+            return None

+        token_expiry = token_details["expires"]

+        token_expiry_time_struct = self.parse_time(token_expiry)

+        if not token_expiry_time_struct:

+            return None

+        token_expiry_time = calendar.timegm(token_expiry_time_struct)

+        token_details["token_expiry_time"] = token_expiry_time

+        ApiAccess.tokens[project_id] = token_details

+        return token_details

+

+    def v2_auth_pwd(self, project):

+        user = ApiAccess.api_config["user"]

+        pwd = ApiAccess.api_config["pwd"]

+        post_body = {

+            "auth": {

+                "passwordCredentials": {

+                    "username": user,

+                    "password": pwd

+                }

+            }

+        }

+        if project is not None:

+            post_body["auth"]["tenantName"] = project

+            project_id = project

+        else:

+            project_id = ""

+        headers = {

+            'Accept': 'application/json',

+            'Content-Type': 'application/json; charset=UTF-8'

+        }

+        return self.v2_auth(project_id, headers, post_body)

+

+    @staticmethod

+    def get_auth_response(project_id):

+        auth_response = ApiAccess.auth_response.get(project_id)

+        if not auth_response:

+            auth_response = ApiAccess.auth_response.get('admin', {})

+        return auth_response

+

+    def get_rel_url(self, relative_url, headers):

+        req_url = ApiAccess.base_url + relative_url

+        return self.get_url(req_url, headers)

+

+    def get_url(self, req_url, headers):

+        response = requests.get(req_url, headers=headers)

+        if response.status_code != requests.codes.ok:

+            # some error happened

+            if "reason" in response:

+                msg = ", reason: {}".format(response.reason)

+            else:

+                msg = ", response: {}".format(response.text)

+            self.log.error("req_url: {} {}".format(req_url, msg))

+            return response

+        ret = response.json()

+        return ret

+

+    def get_region_url(self, region_name, service):

+        if region_name not in self.regions:

+            return None

+        region = self.regions[region_name]

+        s = self.get_service_region_endpoints(region, service)

+        if not s:

+            return None

+        orig_url = s["adminURL"]

+        # replace host name with the host found in config

+        url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url)

+        return url

+

+    # like get_region_url(), but remove everything starting from the "/v2"

+    def get_region_url_nover(self, region, service):

+        full_url = self.get_region_url(region, service)

+        if not full_url:

+            self.log.error("could not find region URL for region: " + region)

+            exit()

+        url = re.sub(r":([0-9]+)/v[2-9].*", r":\1", full_url)

+        return url

+

+    def get_catalog(self, pretty):

+        return jsonify(self.regions, pretty)

+

+    # find the endpoints for a given service name,

+    # considering also alternative service names

+    def get_service_region_endpoints(self, region, service):

+        alternatives = [service]

+        endpoints = region["endpoints"]

+        if service in self.alternative_services:

+            alternatives.extend(self.alternative_services[service])

+        for sname in alternatives:

+            if sname in endpoints:

+                return endpoints[sname]

+        return None

diff --git a/app/discover/fetchers/api/api_fetch_end_points.py b/app/discover/fetchers/api/api_fetch_end_points.py
deleted file mode 100644
index 9471c7e..0000000
--- a/app/discover/fetchers/api/api_fetch_end_points.py
+++ /dev/null
@@ -1,35 +0,0 @@
-###############################################################################
-# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
-# and others                                                                  #
-#                                                                             #
-# All rights reserved. This program and the accompanying materials            #
-# are made available under the terms of the Apache License, Version 2.0       #
-# which accompanies this distribution, and is available at                    #
-# http://www.apache.org/licenses/LICENSE-2.0                                  #
-###############################################################################
-# fetch the end points for a given project (tenant)
-# return list of regions, to allow further recursive scanning
-
-from discover.fetchers.api.api_access import ApiAccess
-
-
-class ApiFetchEndPoints(ApiAccess):
-
-    def get(self, project_id):
-        if project_id != "admin":
-            return []  # XXX currently having problems authenticating to other tenants
-        self.v2_auth_pwd(project_id)
-
-        environment = ApiAccess.config.get_env_name()
-        regions = []
-        services = ApiAccess.auth_response['access']['serviceCatalog']
-        endpoints = []
-        for s in services:
-            if s["type"] != "identity":
-                continue
-            e = s["endpoints"][0]
-            e["environment"] = environment
-            e["project"] = project_id
-            e["type"] = "endpoint"
-            endpoints.append(e)
-        return endpoints
diff --git a/app/discover/fetchers/api/api_fetch_regions.py b/app/discover/fetchers/api/api_fetch_regions.py
index dcc558f..23a3736 100644
--- a/app/discover/fetchers/api/api_fetch_regions.py
+++ b/app/discover/fetchers/api/api_fetch_regions.py
@@ -1,51 +1,53 @@
-###############################################################################
-# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
-# and others                                                                  #
-#                                                                             #
-# All rights reserved. This program and the accompanying materials            #
-# are made available under the terms of the Apache License, Version 2.0       #
-# which accompanies this distribution, and is available at                    #
-# http://www.apache.org/licenses/LICENSE-2.0                                  #
-###############################################################################
-from discover.fetchers.api.api_access import ApiAccess
-
-
-class ApiFetchRegions(ApiAccess):
-    def __init__(self):
-        super(ApiFetchRegions, self).__init__()
-        self.endpoint = ApiAccess.base_url
-
-    def get(self, project_id):
-        token = self.v2_auth_pwd(self.admin_project)
-        if not token:
-            return []
-        # the returned authentication response contains the list of end points
-        # and regions
-        service_catalog = ApiAccess.auth_response.get('access', {}).get('serviceCatalog')
-        if not service_catalog:
-            return []
-        env = self.get_env()
-        ret = []
-        NULL_REGION = "No-Region"
-        for service in service_catalog:
-            for e in service["endpoints"]:
-                if "region" in e:
-                    region_name = e.pop("region")
-                    region_name = region_name if region_name else NULL_REGION
-                else:
-                    region_name = NULL_REGION
-                if region_name in self.regions.keys():
-                    region = self.regions[region_name]
-                else:
-                    region = {
-                        "id": region_name,
-                        "name": region_name,
-                        "endpoints": {}
-                    }
-                    ApiAccess.regions[region_name] = region
-                region["parent_type"] = "regions_folder"
-                region["parent_id"] = env + "-regions"
-                e["service_type"] = service["type"]
-                region["endpoints"][service["name"]] = e
-        ret.extend(list(ApiAccess.regions.values()))
-        return ret
+###############################################################################

+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #

+# and others                                                                  #

+#                                                                             #

+# All rights reserved. This program and the accompanying materials            #

+# are made available under the terms of the Apache License, Version 2.0       #

+# which accompanies this distribution, and is available at                    #

+# http://www.apache.org/licenses/LICENSE-2.0                                  #

+###############################################################################

+from discover.fetchers.api.api_access import ApiAccess

+

+

+class ApiFetchRegions(ApiAccess):

+    def __init__(self):

+        super(ApiFetchRegions, self).__init__()

+        self.endpoint = ApiAccess.base_url

+

+    def get(self, regions_folder_id):

+        token = self.v2_auth_pwd(self.admin_project)

+        if not token:

+            return []

+        # the returned authentication response contains the list of end points

+        # and regions

+        project_id = regions_folder_id.replace('-regions', '')

+        response = ApiAccess.get_auth_response(project_id)

+        service_catalog = response.get('access', {}).get('serviceCatalog')

+        if not service_catalog:

+            return []

+        env = self.get_env()

+        ret = []

+        NULL_REGION = "No-Region"

+        for service in service_catalog:

+            for e in service["endpoints"]:

+                if "region" in e:

+                    region_name = e.pop("region")

+                    region_name = region_name if region_name else NULL_REGION

+                else:

+                    region_name = NULL_REGION

+                if region_name in self.regions.keys():

+                    region = self.regions[region_name]

+                else:

+                    region = {

+                        "id": region_name,

+                        "name": region_name,

+                        "endpoints": {}

+                    }

+                    ApiAccess.regions[region_name] = region

+                region["parent_type"] = "regions_folder"

+                region["parent_id"] = env + "-regions"

+                e["service_type"] = service["type"]

+                region["endpoints"][service["name"]] = e

+        ret.extend(list(ApiAccess.regions.values()))

+        return ret