Update Keystone code from official Github repository with branch Master on 09/01/2015.

Change-Id: I0ff6099e6e2580f87f502002a998bbfe12673498

6 files changed, 193 insertions, 40 deletions

diff --git a/keystone-moon/keystone/identity/backends/ldap.py b/keystone-moon/keystone/identity/backends/ldap.py
index 0f7ee450..7a3cb03b 100644
--- a/keystone-moon/keystone/identity/backends/ldap.py
+++ b/keystone-moon/keystone/identity/backends/ldap.py
@@ -14,13 +14,12 @@
 from __future__ import absolute_import
 import uuid
 
-import ldap
 import ldap.filter
 from oslo_config import cfg
 from oslo_log import log
 import six
 
-from keystone import clean
+from keystone.common import clean
 from keystone.common import driver_hints
 from keystone.common import ldap as common_ldap
 from keystone.common import models
@@ -42,7 +41,7 @@ class Identity(identity.Driver):
         self.group = GroupApi(conf)
 
     def default_assignment_driver(self):
-        return "keystone.assignment.backends.ldap.Assignment"
+        return 'ldap'
 
     def is_domain_aware(self):
         return False
@@ -352,20 +351,18 @@ class GroupApi(common_ldap.BaseLdap):
         """Return a list of groups for which the user is a member."""
 
         user_dn_esc = ldap.filter.escape_filter_chars(user_dn)
-        query = '(&(objectClass=%s)(%s=%s)%s)' % (self.object_class,
-                                                  self.member_attribute,
-                                                  user_dn_esc,
-                                                  self.ldap_filter or '')
+        query = '(%s=%s)%s' % (self.member_attribute,
+                               user_dn_esc,
+                               self.ldap_filter or '')
         return self.get_all(query)
 
     def list_user_groups_filtered(self, user_dn, hints):
         """Return a filtered list of groups for which the user is a member."""
 
         user_dn_esc = ldap.filter.escape_filter_chars(user_dn)
-        query = '(&(objectClass=%s)(%s=%s)%s)' % (self.object_class,
-                                                  self.member_attribute,
-                                                  user_dn_esc,
-                                                  self.ldap_filter or '')
+        query = '(%s=%s)%s' % (self.member_attribute,
+                               user_dn_esc,
+                               self.ldap_filter or '')
         return self.get_all_filtered(hints, query)
 
     def list_group_users(self, group_id):
diff --git a/keystone-moon/keystone/identity/backends/sql.py b/keystone-moon/keystone/identity/backends/sql.py
index 39868416..8bda9a1b 100644
--- a/keystone-moon/keystone/identity/backends/sql.py
+++ b/keystone-moon/keystone/identity/backends/sql.py
@@ -77,7 +77,7 @@ class Identity(identity.Driver):
         super(Identity, self).__init__()
 
     def default_assignment_driver(self):
-        return "keystone.assignment.backends.sql.Assignment"
+        return 'sql'
 
     @property
     def is_sql(self):
@@ -211,28 +211,19 @@ class Identity(identity.Driver):
             session.delete(membership_ref)
 
     def list_groups_for_user(self, user_id, hints):
-        # TODO(henry-nash) We could implement full filtering here by enhancing
-        # the join below.  However, since it is likely to be a fairly rare
-        # occurrence to filter on more than the user_id already being used
-        # here, this is left as future enhancement and until then we leave
-        # it for the controller to do for us.
         session = sql.get_session()
         self.get_user(user_id)
         query = session.query(Group).join(UserGroupMembership)
         query = query.filter(UserGroupMembership.user_id == user_id)
+        query = sql.filter_limit_query(Group, query, hints)
         return [g.to_dict() for g in query]
 
     def list_users_in_group(self, group_id, hints):
-        # TODO(henry-nash) We could implement full filtering here by enhancing
-        # the join below.  However, since it is likely to be a fairly rare
-        # occurrence to filter on more than the group_id already being used
-        # here, this is left as future enhancement and until then we leave
-        # it for the controller to do for us.
         session = sql.get_session()
         self.get_group(group_id)
         query = session.query(User).join(UserGroupMembership)
         query = query.filter(UserGroupMembership.group_id == group_id)
-
+        query = sql.filter_limit_query(User, query, hints)
         return [identity.filter_user(u.to_dict()) for u in query]
 
     def delete_user(self, user_id):
diff --git a/keystone-moon/keystone/identity/controllers.py b/keystone-moon/keystone/identity/controllers.py
index a2676c41..7a6a642a 100644
--- a/keystone-moon/keystone/identity/controllers.py
+++ b/keystone-moon/keystone/identity/controllers.py
@@ -19,8 +19,10 @@ from oslo_log import log
 
 from keystone.common import controller
 from keystone.common import dependency
+from keystone.common import validation
 from keystone import exception
 from keystone.i18n import _, _LW
+from keystone.identity import schema
 from keystone import notifications
 
 
@@ -205,9 +207,8 @@ class UserV3(controller.V3Controller):
         self.check_protection(context, prep_info, ref)
 
     @controller.protected()
+    @validation.validated(schema.user_create, 'user')
     def create_user(self, context, user):
-        self._require_attribute(user, 'name')
-
         # The manager layer will generate the unique ID for users
         ref = self._normalize_dict(user)
         ref = self._normalize_domain_id(context, ref)
@@ -243,6 +244,7 @@ class UserV3(controller.V3Controller):
         return UserV3.wrap_member(context, ref)
 
     @controller.protected()
+    @validation.validated(schema.user_update, 'user')
     def update_user(self, context, user_id, user):
         return self._update_user(context, user_id, user)
 
@@ -291,9 +293,8 @@ class GroupV3(controller.V3Controller):
         self.get_member_from_driver = self.identity_api.get_group
 
     @controller.protected()
+    @validation.validated(schema.group_create, 'group')
     def create_group(self, context, group):
-        self._require_attribute(group, 'name')
-
         # The manager layer will generate the unique ID for groups
         ref = self._normalize_dict(group)
         ref = self._normalize_domain_id(context, ref)
@@ -321,6 +322,7 @@ class GroupV3(controller.V3Controller):
         return GroupV3.wrap_member(context, ref)
 
     @controller.protected()
+    @validation.validated(schema.group_update, 'group')
     def update_group(self, context, group_id, group):
         self._require_matching_id(group_id, group)
         self._require_matching_domain_id(
diff --git a/keystone-moon/keystone/identity/core.py b/keystone-moon/keystone/identity/core.py
index 988df78b..612a1859 100644
--- a/keystone-moon/keystone/identity/core.py
+++ b/keystone-moon/keystone/identity/core.py
@@ -21,11 +21,10 @@ import uuid
 
 from oslo_config import cfg
 from oslo_log import log
-from oslo_utils import importutils
 import six
 
-from keystone import clean
 from keystone.common import cache
+from keystone.common import clean
 from keystone.common import dependency
 from keystone.common import driver_hints
 from keystone.common import manager
@@ -90,8 +89,9 @@ class DomainConfigs(dict):
     _any_sql = False
 
     def _load_driver(self, domain_config):
-        return importutils.import_object(
-            domain_config['cfg'].identity.driver, domain_config['cfg'])
+        return manager.load_driver(Manager.driver_namespace,
+                                   domain_config['cfg'].identity.driver,
+                                   domain_config['cfg'])
 
     def _assert_no_more_than_one_sql_driver(self, domain_id, new_config,
                                             config_file=None):
@@ -111,7 +111,7 @@ class DomainConfigs(dict):
             if not config_file:
                 config_file = _('Database at /domains/%s/config') % domain_id
             raise exception.MultipleSQLDriversInConfig(source=config_file)
-        self._any_sql = new_config['driver'].is_sql
+        self._any_sql = self._any_sql or new_config['driver'].is_sql
 
     def _load_config_from_file(self, resource_api, file_list, domain_name):
 
@@ -176,6 +176,21 @@ class DomainConfigs(dict):
                                   fname)
 
     def _load_config_from_database(self, domain_id, specific_config):
+
+        def _assert_not_sql_driver(domain_id, new_config):
+            """Ensure this is not an sql driver.
+
+            Due to multi-threading safety concerns, we do not currently support
+            the setting of a specific identity driver to sql via the Identity
+            API.
+
+            """
+            if new_config['driver'].is_sql:
+                reason = _('Domain specific sql drivers are not supported via '
+                           'the Identity API. One is specified in '
+                           '/domains/%s/config') % domain_id
+                raise exception.InvalidDomainConfig(reason=reason)
+
         domain_config = {}
         domain_config['cfg'] = cfg.ConfigOpts()
         config.configure(conf=domain_config['cfg'])
@@ -186,10 +201,12 @@ class DomainConfigs(dict):
         for group in specific_config:
             for option in specific_config[group]:
                 domain_config['cfg'].set_override(
-                    option, specific_config[group][option], group)
+                    option, specific_config[group][option],
+                    group, enforce_type=True)
 
+        domain_config['cfg_overrides'] = specific_config
         domain_config['driver'] = self._load_driver(domain_config)
-        self._assert_no_more_than_one_sql_driver(domain_id, domain_config)
+        _assert_not_sql_driver(domain_id, domain_config)
         self[domain_id] = domain_config
 
     def _setup_domain_drivers_from_database(self, standard_driver,
@@ -226,10 +243,12 @@ class DomainConfigs(dict):
                                                   resource_api)
 
     def get_domain_driver(self, domain_id):
+        self.check_config_and_reload_domain_driver_if_required(domain_id)
         if domain_id in self:
             return self[domain_id]['driver']
 
     def get_domain_conf(self, domain_id):
+        self.check_config_and_reload_domain_driver_if_required(domain_id)
         if domain_id in self:
             return self[domain_id]['cfg']
         else:
@@ -249,6 +268,61 @@ class DomainConfigs(dict):
                 # The standard driver
                 self.driver = self.driver()
 
+    def check_config_and_reload_domain_driver_if_required(self, domain_id):
+        """Check for, and load, any new domain specific config for this domain.
+
+        This is only supported for the database-stored domain specific
+        configuration.
+
+        When the domain specific drivers were set up, we stored away the
+        specific config for this domain that was available at that time. So we
+        now read the current version and compare. While this might seem
+        somewhat inefficient, the sensitive config call is cached, so should be
+        light weight. More importantly, when the cache timeout is reached, we
+        will get any config that has been updated from any other keystone
+        process.
+
+        This cache-timeout approach works for both multi-process and
+        multi-threaded keystone configurations. In multi-threaded
+        configurations, even though we might remove a driver object (that
+        could be in use by another thread), this won't actually be thrown away
+        until all references to it have been broken. When that other
+        thread is released back and is restarted with another command to
+        process, next time it accesses the driver it will pickup the new one.
+
+        """
+        if (not CONF.identity.domain_specific_drivers_enabled or
+                not CONF.identity.domain_configurations_from_database):
+            # If specific drivers are not enabled, then there is nothing to do.
+            # If we are not storing the configurations in the database, then
+            # we'll only re-read the domain specific config files on startup
+            # of keystone.
+            return
+
+        latest_domain_config = (
+            self.domain_config_api.
+            get_config_with_sensitive_info(domain_id))
+        domain_config_in_use = domain_id in self
+
+        if latest_domain_config:
+            if (not domain_config_in_use or
+                    latest_domain_config != self[domain_id]['cfg_overrides']):
+                self._load_config_from_database(domain_id,
+                                                latest_domain_config)
+        elif domain_config_in_use:
+            # The domain specific config has been deleted, so should remove the
+            # specific driver for this domain.
+            try:
+                del self[domain_id]
+            except KeyError:
+                # Allow this error in case we are unlucky and in a
+                # multi-threaded situation, two threads happen to be running
+                # in lock step.
+                pass
+        # If we fall into the else condition, this means there is no domain
+        # config set, and there is none in use either, so we have nothing
+        # to do.
+
 
 def domains_configured(f):
     """Wraps API calls to lazy load domain configs after init.
@@ -291,6 +365,7 @@ def exception_translated(exception_type):
     return _exception_translated
 
 
+@notifications.listener
 @dependency.provider('identity_api')
 @dependency.requires('assignment_api', 'credential_api', 'id_mapping_api',
                      'resource_api', 'revoke_api')
@@ -332,6 +407,9 @@ class Manager(manager.Manager):
     mapping by default is a more prudent way to introduce this functionality.
 
     """
+
+    driver_namespace = 'keystone.identity'
+
     _USER = 'user'
     _GROUP = 'group'
 
@@ -521,10 +599,10 @@ class Manager(manager.Manager):
         if (not driver.is_domain_aware() and driver == self.driver and
             domain_id != CONF.identity.default_domain_id and
                 domain_id is not None):
-                    LOG.warning('Found multiple domains being mapped to a '
-                                'driver that does not support that (e.g. '
-                                'LDAP) - Domain ID: %(domain)s, '
-                                'Default Driver: %(driver)s',
+                    LOG.warning(_LW('Found multiple domains being mapped to a '
+                                    'driver that does not support that (e.g. '
+                                    'LDAP) - Domain ID: %(domain)s, '
+                                    'Default Driver: %(driver)s'),
                                 {'domain': domain_id,
                                  'driver': (driver == self.driver)})
                     raise exception.DomainNotFound(domain_id=domain_id)
@@ -765,7 +843,7 @@ class Manager(manager.Manager):
         # Get user details to invalidate the cache.
         user_old = self.get_user(user_id)
         driver.delete_user(entity_id)
-        self.assignment_api.delete_user(user_id)
+        self.assignment_api.delete_user_assignments(user_id)
         self.get_user.invalidate(self, user_id)
         self.get_user_by_name.invalidate(self, user_old['name'],
                                          user_old['domain_id'])
@@ -837,7 +915,7 @@ class Manager(manager.Manager):
         driver.delete_group(entity_id)
         self.get_group.invalidate(self, group_id)
         self.id_mapping_api.delete_id_mapping(group_id)
-        self.assignment_api.delete_group(group_id)
+        self.assignment_api.delete_group_assignments(group_id)
 
         notifications.Audit.deleted(self._GROUP, group_id, initiator)
 
@@ -895,6 +973,19 @@ class Manager(manager.Manager):
         """
         pass
 
+    @notifications.internal(
+        notifications.INVALIDATE_USER_PROJECT_TOKEN_PERSISTENCE)
+    def emit_invalidate_grant_token_persistence(self, user_project):
+        """Emit a notification to the callback system to revoke grant tokens.
+
+        This method and associated callback listener removes the need for
+        making a direct call to another manager to delete and revoke tokens.
+
+        :param user_project: {'user_id': user_id, 'project_id': project_id}
+        :type user_project: dict
+        """
+        pass
+
     @manager.response_truncated
     @domains_configured
     @exception_translated('user')
@@ -1193,6 +1284,8 @@ class Driver(object):
 class MappingManager(manager.Manager):
     """Default pivot point for the ID Mapping backend."""
 
+    driver_namespace = 'keystone.identity.id_mapping'
+
     def __init__(self):
         super(MappingManager, self).__init__(CONF.identity_mapping.driver)
 
diff --git a/keystone-moon/keystone/identity/generator.py b/keystone-moon/keystone/identity/generator.py
index d25426ce..05ad2df5 100644
--- a/keystone-moon/keystone/identity/generator.py
+++ b/keystone-moon/keystone/identity/generator.py
@@ -23,6 +23,7 @@ from keystone.common import dependency
 from keystone.common import manager
 from keystone import exception
 
+
 CONF = cfg.CONF
 
 
@@ -30,6 +31,8 @@ CONF = cfg.CONF
 class Manager(manager.Manager):
     """Default pivot point for the identifier generator backend."""
 
+    driver_namespace = 'keystone.identity.id_generator'
+
     def __init__(self):
         super(Manager, self).__init__(CONF.identity_mapping.generator)
 
diff --git a/keystone-moon/keystone/identity/schema.py b/keystone-moon/keystone/identity/schema.py
new file mode 100644
index 00000000..047fcf02
--- /dev/null
+++ b/keystone-moon/keystone/identity/schema.py
@@ -0,0 +1,67 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.common import validation
+from keystone.common.validation import parameter_types
+
+
+# NOTE(lhcheng): the max length is not applicable since it is specific
+# to the SQL backend, LDAP does not have length limitation.
+_identity_name = {
+    'type': 'string',
+    'minLength': 1
+}
+
+_user_properties = {
+    'default_project_id': validation.nullable(parameter_types.id_string),
+    'description': validation.nullable(parameter_types.description),
+    'domain_id': parameter_types.id_string,
+    'enabled': parameter_types.boolean,
+    'name': _identity_name,
+    'password': {
+        'type': ['string', 'null']
+    }
+}
+
+user_create = {
+    'type': 'object',
+    'properties': _user_properties,
+    'required': ['name'],
+    'additionalProperties': True
+}
+
+user_update = {
+    'type': 'object',
+    'properties': _user_properties,
+    'minProperties': 1,
+    'additionalProperties': True
+}
+
+_group_properties = {
+    'description': validation.nullable(parameter_types.description),
+    'domain_id': parameter_types.id_string,
+    'name': _identity_name
+}
+
+group_create = {
+    'type': 'object',
+    'properties': _group_properties,
+    'required': ['name'],
+    'additionalProperties': True
+}
+
+group_update = {
+    'type': 'object',
+    'properties': _group_properties,
+    'minProperties': 1,
+    'additionalProperties': True
+}