Update Keystone code from official Github repository with branch Master on 09/01/2015.

Change-Id: I0ff6099e6e2580f87f502002a998bbfe12673498

5 files changed, 159 insertions, 41 deletions

diff --git a/keystone-moon/keystone/catalog/backends/sql.py b/keystone-moon/keystone/catalog/backends/sql.py
index 8ab82305..0db6d498 100644
--- a/keystone-moon/keystone/catalog/backends/sql.py
+++ b/keystone-moon/keystone/catalog/backends/sql.py
@@ -16,7 +16,6 @@
 import itertools
 
 from oslo_config import cfg
-import six
 import sqlalchemy
 from sqlalchemy.sql import true
 
@@ -269,10 +268,28 @@ class Catalog(catalog.Driver):
         return ref.to_dict()
 
     def get_catalog(self, user_id, tenant_id):
+        """Retrieve and format the V2 service catalog.
+
+        :param user_id: The id of the user who has been authenticated for
+            creating service catalog.
+        :param tenant_id: The id of the project. 'tenant_id' will be None
+            in the case this being called to create a catalog to go in a
+            domain scoped token. In this case, any endpoint that requires
+            a tenant_id as part of their URL will be skipped (as would a whole
+            service if, as a consequence, it has no valid endpoints).
+
+        :returns: A nested dict representing the service catalog or an
+                  empty dict.
+
+        """
         substitutions = dict(
-            itertools.chain(six.iteritems(CONF),
-                            six.iteritems(CONF.eventlet_server)))
-        substitutions.update({'tenant_id': tenant_id, 'user_id': user_id})
+            itertools.chain(CONF.items(), CONF.eventlet_server.items()))
+        substitutions.update({'user_id': user_id})
+        silent_keyerror_failures = []
+        if tenant_id:
+            substitutions.update({'tenant_id': tenant_id})
+        else:
+            silent_keyerror_failures = ['tenant_id']
 
         session = sql.get_session()
         endpoints = (session.query(Endpoint).
@@ -285,7 +302,13 @@ class Catalog(catalog.Driver):
             if not endpoint.service['enabled']:
                 continue
             try:
-                url = core.format_url(endpoint['url'], substitutions)
+                formatted_url = core.format_url(
+                    endpoint['url'], substitutions,
+                    silent_keyerror_failures=silent_keyerror_failures)
+                if formatted_url is not None:
+                    url = formatted_url
+                else:
+                    continue
             except exception.MalformedEndpoint:
                 continue  # this failure is already logged in format_url()
 
@@ -304,11 +327,26 @@ class Catalog(catalog.Driver):
         return catalog
 
     def get_v3_catalog(self, user_id, tenant_id):
+        """Retrieve and format the current V3 service catalog.
+
+        :param user_id: The id of the user who has been authenticated for
+            creating service catalog.
+        :param tenant_id: The id of the project. 'tenant_id' will be None in
+            the case this being called to create a catalog to go in a domain
+            scoped token. In this case, any endpoint that requires a
+            tenant_id as part of their URL will be skipped.
+
+        :returns: A list representing the service catalog or an empty list
+
+        """
         d = dict(
-            itertools.chain(six.iteritems(CONF),
-                            six.iteritems(CONF.eventlet_server)))
-        d.update({'tenant_id': tenant_id,
-                  'user_id': user_id})
+            itertools.chain(CONF.items(), CONF.eventlet_server.items()))
+        d.update({'user_id': user_id})
+        silent_keyerror_failures = []
+        if tenant_id:
+            d.update({'tenant_id': tenant_id})
+        else:
+            silent_keyerror_failures = ['tenant_id']
 
         session = sql.get_session()
         services = (session.query(Service).filter(Service.enabled == true()).
@@ -322,12 +360,20 @@ class Catalog(catalog.Driver):
                 del endpoint['enabled']
                 endpoint['region'] = endpoint['region_id']
                 try:
-                    endpoint['url'] = core.format_url(endpoint['url'], d)
+                    formatted_url = core.format_url(
+                        endpoint['url'], d,
+                        silent_keyerror_failures=silent_keyerror_failures)
+                    if formatted_url:
+                        endpoint['url'] = formatted_url
+                    else:
+                        continue
                 except exception.MalformedEndpoint:
                     continue  # this failure is already logged in format_url()
 
                 yield endpoint
 
+        # TODO(davechen): If there is service with no endpoints, we should skip
+        # the service instead of keeping it in the catalog, see bug #1436704.
         def make_v3_service(svc):
             eps = list(make_v3_endpoints(svc.endpoints))
             service = {'endpoints': eps, 'id': svc.id, 'type': svc.type}
diff --git a/keystone-moon/keystone/catalog/backends/templated.py b/keystone-moon/keystone/catalog/backends/templated.py
index d3ee105d..31d8b9e0 100644
--- a/keystone-moon/keystone/catalog/backends/templated.py
+++ b/keystone-moon/keystone/catalog/backends/templated.py
@@ -17,7 +17,6 @@ import os.path
 
 from oslo_config import cfg
 from oslo_log import log
-import six
 
 from keystone.catalog.backends import kvs
 from keystone.catalog import core
@@ -107,19 +106,43 @@ class Catalog(kvs.Catalog):
             raise
 
     def get_catalog(self, user_id, tenant_id):
+        """Retrieve and format the V2 service catalog.
+
+        :param user_id: The id of the user who has been authenticated for
+            creating service catalog.
+        :param tenant_id: The id of the project. 'tenant_id' will be None in
+            the case this being called to create a catalog to go in a domain
+            scoped token. In this case, any endpoint that requires a tenant_id
+            as part of their URL will be skipped.
+
+        :returns: A nested dict representing the service catalog or an
+                  empty dict.
+
+        """
         substitutions = dict(
-            itertools.chain(six.iteritems(CONF),
-                            six.iteritems(CONF.eventlet_server)))
-        substitutions.update({'tenant_id': tenant_id, 'user_id': user_id})
+            itertools.chain(CONF.items(), CONF.eventlet_server.items()))
+        substitutions.update({'user_id': user_id})
+        silent_keyerror_failures = []
+        if tenant_id:
+            substitutions.update({'tenant_id': tenant_id})
+        else:
+            silent_keyerror_failures = ['tenant_id']
 
         catalog = {}
-        for region, region_ref in six.iteritems(self.templates):
+        # TODO(davechen): If there is service with no endpoints, we should
+        # skip the service instead of keeping it in the catalog.
+        # see bug #1436704.
+        for region, region_ref in self.templates.items():
             catalog[region] = {}
-            for service, service_ref in six.iteritems(region_ref):
+            for service, service_ref in region_ref.items():
                 service_data = {}
                 try:
-                    for k, v in six.iteritems(service_ref):
-                        service_data[k] = core.format_url(v, substitutions)
+                    for k, v in service_ref.items():
+                        formatted_value = core.format_url(
+                            v, substitutions,
+                            silent_keyerror_failures=silent_keyerror_failures)
+                        if formatted_value:
+                            service_data[k] = formatted_value
                 except exception.MalformedEndpoint:
                     continue  # this failure is already logged in format_url()
                 catalog[region][service] = service_data
diff --git a/keystone-moon/keystone/catalog/controllers.py b/keystone-moon/keystone/catalog/controllers.py
index 3518c4bf..92046e8a 100644
--- a/keystone-moon/keystone/catalog/controllers.py
+++ b/keystone-moon/keystone/catalog/controllers.py
@@ -15,8 +15,7 @@
 
 import uuid
 
-import six
-
+from keystone.catalog import core
 from keystone.catalog import schema
 from keystone.common import controller
 from keystone.common import dependency
@@ -88,7 +87,7 @@ class Endpoint(controller.V2Controller):
 
             # add the legacy endpoint with an interface url
             legacy_ep['%surl' % endpoint['interface']] = endpoint['url']
-        return {'endpoints': legacy_endpoints.values()}
+        return {'endpoints': list(legacy_endpoints.values())}
 
     @controller.v2_deprecated
     def create_endpoint(self, context, endpoint):
@@ -100,6 +99,14 @@ class Endpoint(controller.V2Controller):
         # service_id is necessary
         self._require_attribute(endpoint, 'service_id')
 
+        # we should check publicurl, adminurl, internalurl
+        # if invalid, we should raise an exception to reject
+        # the request
+        for interface in INTERFACES:
+            interface_url = endpoint.get(interface + 'url')
+            if interface_url:
+                core.check_endpoint_url(interface_url)
+
         initiator = notifications._get_request_audit_info(context)
 
         if endpoint.get('region') is not None:
@@ -124,7 +131,7 @@ class Endpoint(controller.V2Controller):
                 legacy_endpoint_ref.pop(url)
 
         legacy_endpoint_id = uuid.uuid4().hex
-        for interface, url in six.iteritems(urls):
+        for interface, url in urls.items():
             endpoint_ref = endpoint.copy()
             endpoint_ref['id'] = uuid.uuid4().hex
             endpoint_ref['legacy_endpoint_id'] = legacy_endpoint_id
@@ -301,13 +308,14 @@ class EndpointV3(controller.V3Controller):
     @controller.protected()
     @validation.validated(schema.endpoint_create, 'endpoint')
     def create_endpoint(self, context, endpoint):
+        core.check_endpoint_url(endpoint['url'])
         ref = self._assign_unique_id(self._normalize_dict(endpoint))
         ref = self._validate_endpoint_region(ref, context)
         initiator = notifications._get_request_audit_info(context)
         ref = self.catalog_api.create_endpoint(ref['id'], ref, initiator)
         return EndpointV3.wrap_member(context, ref)
 
-    @controller.filterprotected('interface', 'service_id')
+    @controller.filterprotected('interface', 'service_id', 'region_id')
     def list_endpoints(self, context, filters):
         hints = EndpointV3.build_driver_hints(context, filters)
         refs = self.catalog_api.list_endpoints(hints=hints)
diff --git a/keystone-moon/keystone/catalog/core.py b/keystone-moon/keystone/catalog/core.py
index fba26b89..6883b024 100644
--- a/keystone-moon/keystone/catalog/core.py
+++ b/keystone-moon/keystone/catalog/core.py
@@ -16,6 +16,7 @@
 """Main entry point into the Catalog service."""
 
 import abc
+import itertools
 
 from oslo_config import cfg
 from oslo_log import log
@@ -35,25 +36,27 @@ from keystone import notifications
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)
 MEMOIZE = cache.get_memoization_decorator(section='catalog')
+WHITELISTED_PROPERTIES = [
+    'tenant_id', 'user_id', 'public_bind_host', 'admin_bind_host',
+    'compute_host', 'admin_port', 'public_port',
+    'public_endpoint', 'admin_endpoint', ]
 
 
-def format_url(url, substitutions):
+def format_url(url, substitutions, silent_keyerror_failures=None):
     """Formats a user-defined URL with the given substitutions.
 
     :param string url: the URL to be formatted
     :param dict substitutions: the dictionary used for substitution
+    :param list silent_keyerror_failures: keys for which we should be silent
+        if there is a KeyError exception on substitution attempt
     :returns: a formatted URL
 
     """
 
-    WHITELISTED_PROPERTIES = [
-        'tenant_id', 'user_id', 'public_bind_host', 'admin_bind_host',
-        'compute_host', 'compute_port', 'admin_port', 'public_port',
-        'public_endpoint', 'admin_endpoint', ]
-
     substitutions = utils.WhiteListedItemFilter(
         WHITELISTED_PROPERTIES,
         substitutions)
+    allow_keyerror = silent_keyerror_failures or []
     try:
         result = url.replace('$(', '%(') % substitutions
     except AttributeError:
@@ -61,10 +64,14 @@ def format_url(url, substitutions):
                   {"url": url})
         raise exception.MalformedEndpoint(endpoint=url)
     except KeyError as e:
-        LOG.error(_LE("Malformed endpoint %(url)s - unknown key %(keyerror)s"),
-                  {"url": url,
-                   "keyerror": e})
-        raise exception.MalformedEndpoint(endpoint=url)
+        if not e.args or e.args[0] not in allow_keyerror:
+            LOG.error(_LE("Malformed endpoint %(url)s - unknown key "
+                          "%(keyerror)s"),
+                      {"url": url,
+                       "keyerror": e})
+            raise exception.MalformedEndpoint(endpoint=url)
+        else:
+            result = None
     except TypeError as e:
         LOG.error(_LE("Malformed endpoint '%(url)s'. The following type error "
                       "occurred during string substitution: %(typeerror)s"),
@@ -78,6 +85,28 @@ def format_url(url, substitutions):
     return result
 
 
+def check_endpoint_url(url):
+    """Check substitution of url.
+
+    The invalid urls are as follows:
+    urls with substitutions that is not in the whitelist
+
+    Check the substitutions in the URL to make sure they are valid
+    and on the whitelist.
+
+    :param str url: the URL to validate
+    :rtype: None
+    :raises keystone.exception.URLValidationError: if the URL is invalid
+    """
+    # check whether the property in the path is exactly the same
+    # with that in the whitelist below
+    substitutions = dict(zip(WHITELISTED_PROPERTIES, itertools.repeat('')))
+    try:
+        url.replace('$(', '%(') % substitutions
+    except (KeyError, TypeError, ValueError):
+        raise exception.URLValidationError(url)
+
+
 @dependency.provider('catalog_api')
 class Manager(manager.Manager):
     """Default pivot point for the Catalog backend.
@@ -86,6 +115,9 @@ class Manager(manager.Manager):
     dynamically calls the backend.
 
     """
+
+    driver_namespace = 'keystone.catalog'
+
     _ENDPOINT = 'endpoint'
     _SERVICE = 'service'
     _REGION = 'region'
@@ -103,10 +135,12 @@ class Manager(manager.Manager):
             msg = _('Duplicate ID, %s.') % region_ref['id']
             raise exception.Conflict(type='region', details=msg)
 
-        # NOTE(lbragstad): The description column of the region database
-        # can not be null. So if the user doesn't pass in a description then
-        # set it to an empty string.
-        region_ref.setdefault('description', '')
+        # NOTE(lbragstad,dstanek): The description column of the region
+        # database cannot be null. So if the user doesn't pass in a
+        # description or passes in a null description then set it to an
+        # empty string.
+        if region_ref.get('description') is None:
+            region_ref['description'] = ''
         try:
             ret = self.driver.create_region(region_ref)
         except exception.NotFound:
@@ -124,6 +158,11 @@ class Manager(manager.Manager):
             raise exception.RegionNotFound(region_id=region_id)
 
     def update_region(self, region_id, region_ref, initiator=None):
+        # NOTE(lbragstad,dstanek): The description column of the region
+        # database cannot be null. So if the user passes in a null
+        # description set it to an empty string.
+        if 'description' in region_ref and region_ref['description'] is None:
+            region_ref['description'] = ''
         ref = self.driver.update_region(region_id, region_ref)
         notifications.Audit.updated(self._REGION, region_id, initiator)
         self.get_region.invalidate(self, region_id)
@@ -475,14 +514,14 @@ class Driver(object):
         v2_catalog = self.get_catalog(user_id, tenant_id)
         v3_catalog = []
 
-        for region_name, region in six.iteritems(v2_catalog):
-            for service_type, service in six.iteritems(region):
+        for region_name, region in v2_catalog.items():
+            for service_type, service in region.items():
                 service_v3 = {
                     'type': service_type,
                     'endpoints': []
                 }
 
-                for attr, value in six.iteritems(service):
+                for attr, value in service.items():
                     # Attributes that end in URL are interfaces. In the V2
                     # catalog, these are internalURL, publicURL, and adminURL.
                     # For example, <region_name>.publicURL=<URL> in the V2
diff --git a/keystone-moon/keystone/catalog/schema.py b/keystone-moon/keystone/catalog/schema.py
index a779ad02..671f1233 100644
--- a/keystone-moon/keystone/catalog/schema.py
+++ b/keystone-moon/keystone/catalog/schema.py
@@ -14,7 +14,9 @@ from keystone.common.validation import parameter_types
 
 
 _region_properties = {
-    'description': parameter_types.description,
+    'description': {
+        'type': ['string', 'null'],
+    },
     # NOTE(lbragstad): Regions use ID differently. The user can specify the ID
     # or it will be generated automatically.
     'id': {