Update Keystone code from official Github repository with branch Master on 09/01/2015.

Change-Id: I0ff6099e6e2580f87f502002a998bbfe12673498

5 files changed, 131 insertions, 56 deletions

diff --git a/keystone-moon/keystone/resource/backends/ldap.py b/keystone-moon/keystone/resource/backends/ldap.py
index 434c2b04..43684035 100644
--- a/keystone-moon/keystone/resource/backends/ldap.py
+++ b/keystone-moon/keystone/resource/backends/ldap.py
@@ -17,7 +17,7 @@ import uuid
 from oslo_config import cfg
 from oslo_log import log
 
-from keystone import clean
+from keystone.common import clean
 from keystone.common import driver_hints
 from keystone.common import ldap as common_ldap
 from keystone.common import models
@@ -47,7 +47,7 @@ class Resource(resource.Driver):
         self.project = ProjectApi(CONF)
 
     def default_assignment_driver(self):
-        return 'keystone.assignment.backends.ldap.Assignment'
+        return 'ldap'
 
     def _set_default_parent_project(self, ref):
         """If the parent project ID has not been set, set it to None."""
@@ -60,6 +60,14 @@ class Resource(resource.Driver):
         else:
             raise ValueError(_('Expected dict or list: %s') % type(ref))
 
+    def _set_default_is_domain_project(self, ref):
+        if isinstance(ref, dict):
+            return dict(ref, is_domain=False)
+        elif isinstance(ref, list):
+            return [self._set_default_is_domain_project(x) for x in ref]
+        else:
+            raise ValueError(_('Expected dict or list: %s') % type(ref))
+
     def _validate_parent_project_is_none(self, ref):
         """If a parent_id different from None was given,
            raises InvalidProjectException.
@@ -69,8 +77,15 @@ class Resource(resource.Driver):
         if parent_id is not None:
             raise exception.InvalidParentProject(parent_id)
 
+    def _validate_is_domain_field_is_false(self, ref):
+        is_domain = ref.pop('is_domain', None)
+        if is_domain:
+            raise exception.ValidationError(_('LDAP does not support projects '
+                                              'with is_domain flag enabled'))
+
     def _set_default_attributes(self, project_ref):
         project_ref = self._set_default_domain(project_ref)
+        project_ref = self._set_default_is_domain_project(project_ref)
         return self._set_default_parent_project(project_ref)
 
     def get_project(self, tenant_id):
@@ -116,8 +131,8 @@ class Resource(resource.Driver):
 
     def create_project(self, tenant_id, tenant):
         self.project.check_allow_create()
-        tenant = self._validate_default_domain(tenant)
         self._validate_parent_project_is_none(tenant)
+        self._validate_is_domain_field_is_false(tenant)
         tenant['name'] = clean.project_name(tenant['name'])
         data = tenant.copy()
         if 'id' not in data or data['id'] is None:
@@ -130,6 +145,7 @@ class Resource(resource.Driver):
     def update_project(self, tenant_id, tenant):
         self.project.check_allow_update()
         tenant = self._validate_default_domain(tenant)
+        self._validate_is_domain_field_is_false(tenant)
         if 'name' in tenant:
             tenant['name'] = clean.project_name(tenant['name'])
         return self._set_default_attributes(
diff --git a/keystone-moon/keystone/resource/backends/sql.py b/keystone-moon/keystone/resource/backends/sql.py
index fb117240..3a0d8cea 100644
--- a/keystone-moon/keystone/resource/backends/sql.py
+++ b/keystone-moon/keystone/resource/backends/sql.py
@@ -13,7 +13,7 @@
 from oslo_config import cfg
 from oslo_log import log
 
-from keystone import clean
+from keystone.common import clean
 from keystone.common import sql
 from keystone import exception
 from keystone.i18n import _LE
@@ -27,7 +27,7 @@ LOG = log.getLogger(__name__)
 class Resource(keystone_resource.Driver):
 
     def default_assignment_driver(self):
-        return 'keystone.assignment.backends.sql.Assignment'
+        return 'sql'
 
     def _get_project(self, session, project_id):
         project_ref = session.query(Project).get(project_id)
@@ -91,10 +91,9 @@ class Resource(keystone_resource.Driver):
 
     def list_projects_in_subtree(self, project_id):
         with sql.transaction() as session:
-            project = self._get_project(session, project_id).to_dict()
-            children = self._get_children(session, [project['id']])
+            children = self._get_children(session, [project_id])
             subtree = []
-            examined = set(project['id'])
+            examined = set([project_id])
             while children:
                 children_ids = set()
                 for ref in children:
@@ -106,7 +105,7 @@ class Resource(keystone_resource.Driver):
                         return
                     children_ids.add(ref['id'])
 
-                examined.union(children_ids)
+                examined.update(children_ids)
                 subtree += children
                 children = self._get_children(session, children_ids)
             return subtree
@@ -246,7 +245,7 @@ class Domain(sql.ModelBase, sql.DictBase):
 class Project(sql.ModelBase, sql.DictBase):
     __tablename__ = 'project'
     attributes = ['id', 'name', 'domain_id', 'description', 'enabled',
-                  'parent_id']
+                  'parent_id', 'is_domain']
     id = sql.Column(sql.String(64), primary_key=True)
     name = sql.Column(sql.String(64), nullable=False)
     domain_id = sql.Column(sql.String(64), sql.ForeignKey('domain.id'),
@@ -255,6 +254,7 @@ class Project(sql.ModelBase, sql.DictBase):
     enabled = sql.Column(sql.Boolean)
     extra = sql.Column(sql.JsonBlob())
     parent_id = sql.Column(sql.String(64), sql.ForeignKey('project.id'))
+    is_domain = sql.Column(sql.Boolean, default=False, nullable=False)
     # Unique constraint across two columns to create the separation
     # rather than just only 'name' being unique
     __table_args__ = (sql.UniqueConstraint('domain_id', 'name'), {})
diff --git a/keystone-moon/keystone/resource/controllers.py b/keystone-moon/keystone/resource/controllers.py
index 886b5eb1..60c4e025 100644
--- a/keystone-moon/keystone/resource/controllers.py
+++ b/keystone-moon/keystone/resource/controllers.py
@@ -47,27 +47,37 @@ class Tenant(controller.V2Controller):
         self.assert_admin(context)
         tenant_refs = self.resource_api.list_projects_in_domain(
             CONF.identity.default_domain_id)
-        for tenant_ref in tenant_refs:
-            tenant_ref = self.filter_domain_id(tenant_ref)
+        tenant_refs = [self.v3_to_v2_project(tenant_ref)
+                       for tenant_ref in tenant_refs
+                       if not tenant_ref.get('is_domain')]
         params = {
             'limit': context['query_string'].get('limit'),
             'marker': context['query_string'].get('marker'),
         }
         return self.format_project_list(tenant_refs, **params)
 
+    def _assert_not_is_domain_project(self, project_id, project_ref=None):
+        # Projects acting as a domain should not be visible via v2
+        if not project_ref:
+            project_ref = self.resource_api.get_project(project_id)
+        if project_ref.get('is_domain'):
+            raise exception.ProjectNotFound(project_id)
+
     @controller.v2_deprecated
     def get_project(self, context, tenant_id):
         # TODO(termie): this stuff should probably be moved to middleware
         self.assert_admin(context)
         ref = self.resource_api.get_project(tenant_id)
-        return {'tenant': self.filter_domain_id(ref)}
+        self._assert_not_is_domain_project(tenant_id, ref)
+        return {'tenant': self.v3_to_v2_project(ref)}
 
     @controller.v2_deprecated
     def get_project_by_name(self, context, tenant_name):
         self.assert_admin(context)
+        # Projects acting as a domain should not be visible via v2
         ref = self.resource_api.get_project_by_name(
             tenant_name, CONF.identity.default_domain_id)
-        return {'tenant': self.filter_domain_id(ref)}
+        return {'tenant': self.v3_to_v2_project(ref)}
 
     # CRUD Extension
     @controller.v2_deprecated
@@ -83,23 +93,25 @@ class Tenant(controller.V2Controller):
         tenant = self.resource_api.create_project(
             tenant_ref['id'],
             self._normalize_domain_id(context, tenant_ref))
-        return {'tenant': self.filter_domain_id(tenant)}
+        return {'tenant': self.v3_to_v2_project(tenant)}
 
     @controller.v2_deprecated
     def update_project(self, context, tenant_id, tenant):
         self.assert_admin(context)
-        # Remove domain_id if specified - a v2 api caller should not
-        # be specifying that
+        self._assert_not_is_domain_project(tenant_id)
+        # Remove domain_id and is_domain if specified - a v2 api caller
+        # should not be specifying that
         clean_tenant = tenant.copy()
         clean_tenant.pop('domain_id', None)
-
+        clean_tenant.pop('is_domain', None)
         tenant_ref = self.resource_api.update_project(
             tenant_id, clean_tenant)
-        return {'tenant': tenant_ref}
+        return {'tenant': self.v3_to_v2_project(tenant_ref)}
 
     @controller.v2_deprecated
     def delete_project(self, context, tenant_id):
         self.assert_admin(context)
+        self._assert_not_is_domain_project(tenant_id)
         self.resource_api.delete_project(tenant_id)
 
 
@@ -201,9 +213,18 @@ class ProjectV3(controller.V3Controller):
     def create_project(self, context, project):
         ref = self._assign_unique_id(self._normalize_dict(project))
         ref = self._normalize_domain_id(context, ref)
+
+        if ref.get('is_domain'):
+            msg = _('The creation of projects acting as domains is not '
+                    'allowed yet.')
+            raise exception.NotImplemented(msg)
+
         initiator = notifications._get_request_audit_info(context)
-        ref = self.resource_api.create_project(ref['id'], ref,
-                                               initiator=initiator)
+        try:
+            ref = self.resource_api.create_project(ref['id'], ref,
+                                                   initiator=initiator)
+        except exception.DomainNotFound as e:
+            raise exception.ValidationError(e)
         return ProjectV3.wrap_member(context, ref)
 
     @controller.filterprotected('domain_id', 'enabled', 'name',
diff --git a/keystone-moon/keystone/resource/core.py b/keystone-moon/keystone/resource/core.py
index 017eb4e7..ca69b729 100644
--- a/keystone-moon/keystone/resource/core.py
+++ b/keystone-moon/keystone/resource/core.py
@@ -10,7 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-"""Main entry point into the resource service."""
+"""Main entry point into the Resource service."""
 
 import abc
 
@@ -18,12 +18,11 @@ from oslo_config import cfg
 from oslo_log import log
 import six
 
-from keystone import clean
 from keystone.common import cache
+from keystone.common import clean
 from keystone.common import dependency
 from keystone.common import driver_hints
 from keystone.common import manager
-from keystone.contrib import federation
 from keystone import exception
 from keystone.i18n import _, _LE, _LW
 from keystone import notifications
@@ -47,12 +46,15 @@ def calc_default_domain():
 @dependency.requires('assignment_api', 'credential_api', 'domain_config_api',
                      'identity_api', 'revoke_api')
 class Manager(manager.Manager):
-    """Default pivot point for the resource backend.
+    """Default pivot point for the Resource backend.
 
     See :mod:`keystone.common.manager.Manager` for more details on how this
     dynamically calls the backend.
 
     """
+
+    driver_namespace = 'keystone.resource'
+
     _DOMAIN = 'domain'
     _PROJECT = 'project'
 
@@ -62,9 +64,8 @@ class Manager(manager.Manager):
         resource_driver = CONF.resource.driver
 
         if resource_driver is None:
-            assignment_driver = (
-                dependency.get_provider('assignment_api').driver)
-            resource_driver = assignment_driver.default_resource_driver()
+            assignment_manager = dependency.get_provider('assignment_api')
+            resource_driver = assignment_manager.default_resource_driver()
 
         super(Manager, self).__init__(resource_driver)
 
@@ -86,21 +87,23 @@ class Manager(manager.Manager):
         tenant['enabled'] = clean.project_enabled(tenant['enabled'])
         tenant.setdefault('description', '')
         tenant.setdefault('parent_id', None)
+        tenant.setdefault('is_domain', False)
 
+        self.get_domain(tenant.get('domain_id'))
         if tenant.get('parent_id') is not None:
             parent_ref = self.get_project(tenant.get('parent_id'))
             parents_list = self.list_project_parents(parent_ref['id'])
             parents_list.append(parent_ref)
             for ref in parents_list:
                 if ref.get('domain_id') != tenant.get('domain_id'):
-                    raise exception.ForbiddenAction(
-                        action=_('cannot create a project within a different '
-                                 'domain than its parents.'))
+                    raise exception.ValidationError(
+                        message=_('cannot create a project within a different '
+                                  'domain than its parents.'))
                 if not ref.get('enabled', True):
-                    raise exception.ForbiddenAction(
-                        action=_('cannot create a project in a '
-                                 'branch containing a disabled '
-                                 'project: %s') % ref['id'])
+                    raise exception.ValidationError(
+                        message=_('cannot create a project in a '
+                                  'branch containing a disabled '
+                                  'project: %s') % ref['id'])
             self._assert_max_hierarchy_depth(tenant.get('parent_id'),
                                              parents_list)
 
@@ -135,14 +138,13 @@ class Manager(manager.Manager):
         """
         # NOTE(marek-denis): We cannot create this attribute in the __init__ as
         # config values are always initialized to default value.
-        federated_domain = (CONF.federation.federated_domain_name or
-                            federation.FEDERATED_DOMAIN_KEYWORD).lower()
+        federated_domain = CONF.federation.federated_domain_name.lower()
         if (domain.get('name') and domain['name'].lower() == federated_domain):
             raise AssertionError(_('Domain cannot be named %s')
-                                 % federated_domain)
+                                 % domain['name'])
         if (domain_id.lower() == federated_domain):
             raise AssertionError(_('Domain cannot have ID %s')
-                                 % federated_domain)
+                                 % domain_id)
 
     def assert_project_enabled(self, project_id, project=None):
         """Assert the project is enabled and its associated domain is enabled.
@@ -177,7 +179,7 @@ class Manager(manager.Manager):
                              'disabled parents') % project_id)
 
     def _assert_whole_subtree_is_disabled(self, project_id):
-        subtree_list = self.driver.list_projects_in_subtree(project_id)
+        subtree_list = self.list_projects_in_subtree(project_id)
         for ref in subtree_list:
             if ref.get('enabled', True):
                 raise exception.ForbiddenAction(
@@ -194,6 +196,11 @@ class Manager(manager.Manager):
             raise exception.ForbiddenAction(
                 action=_('Update of `parent_id` is not allowed.'))
 
+        if ('is_domain' in tenant and
+                tenant['is_domain'] != original_tenant['is_domain']):
+            raise exception.ValidationError(
+                message=_('Update of `is_domain` is not allowed.'))
+
         if 'enabled' in tenant:
             tenant['enabled'] = clean.project_enabled(tenant['enabled'])
 
@@ -241,15 +248,23 @@ class Manager(manager.Manager):
         user_projects = self.assignment_api.list_projects_for_user(user_id)
         user_projects_ids = set([proj['id'] for proj in user_projects])
         # Keep only the projects present in user_projects
-        projects_list = [proj for proj in projects_list
-                         if proj['id'] in user_projects_ids]
+        return [proj for proj in projects_list
+                if proj['id'] in user_projects_ids]
+
+    def _assert_valid_project_id(self, project_id):
+        if project_id is None:
+            msg = _('Project field is required and cannot be empty.')
+            raise exception.ValidationError(message=msg)
+        # Check if project_id exists
+        self.get_project(project_id)
 
     def list_project_parents(self, project_id, user_id=None):
+        self._assert_valid_project_id(project_id)
         parents = self.driver.list_project_parents(project_id)
         # If a user_id was provided, the returned list should be filtered
         # against the projects this user has access to.
         if user_id:
-            self._filter_projects_list(parents, user_id)
+            parents = self._filter_projects_list(parents, user_id)
         return parents
 
     def _build_parents_as_ids_dict(self, project, parents_by_id):
@@ -296,11 +311,12 @@ class Manager(manager.Manager):
         return parents_as_ids
 
     def list_projects_in_subtree(self, project_id, user_id=None):
+        self._assert_valid_project_id(project_id)
         subtree = self.driver.list_projects_in_subtree(project_id)
         # If a user_id was provided, the returned list should be filtered
         # against the projects this user has access to.
         if user_id:
-            self._filter_projects_list(subtree, user_id)
+            subtree = self._filter_projects_list(subtree, user_id)
         return subtree
 
     def _build_subtree_as_ids_dict(self, project_id, subtree_by_parent):
@@ -780,6 +796,9 @@ class Driver(object):
             raise exception.DomainNotFound(domain_id=domain_id)
 
 
+MEMOIZE_CONFIG = cache.get_memoization_decorator(section='domain_config')
+
+
 @dependency.provider('domain_config_api')
 class DomainConfigManager(manager.Manager):
     """Default pivot point for the Domain Config backend."""
@@ -793,6 +812,8 @@ class DomainConfigManager(manager.Manager):
     # Only those options that affect the domain-specific driver support in
     # the identity manager are supported.
 
+    driver_namespace = 'keystone.resource.domain_config'
+
     whitelisted_options = {
         'identity': ['driver'],
         'ldap': [
@@ -975,6 +996,10 @@ class DomainConfigManager(manager.Manager):
             self.create_config_option(
                 domain_id, option['group'], option['option'], option['value'],
                 sensitive=True)
+        # Since we are caching on the full substituted config, we just
+        # invalidate here, rather than try and create the right result to
+        # cache.
+        self.get_config_with_sensitive_info.invalidate(self, domain_id)
         return self._list_to_config(whitelisted)
 
     def get_config(self, domain_id, group=None, option=None):
@@ -999,7 +1024,7 @@ class DomainConfigManager(manager.Manager):
                     'url': 'myurl'
                     'user_tree_dn': 'OU=myou'},
                 'identity': {
-                    'driver': 'keystone.identity.backends.ldap.Identity'}
+                    'driver': 'ldap'}
 
             }
 
@@ -1077,22 +1102,22 @@ class DomainConfigManager(manager.Manager):
                             'provided contains group %(group_other)s '
                             'instead') % {
                                 'group': group,
-                                'group_other': config.keys()[0]}
+                                'group_other': list(config.keys())[0]}
                     raise exception.InvalidDomainConfig(reason=msg)
                 if option and option not in config[group]:
                     msg = _('Trying to update option %(option)s in group '
                             '%(group)s, but config provided contains option '
                             '%(option_other)s instead') % {
                                 'group': group, 'option': option,
-                                'option_other': config[group].keys()[0]}
+                                'option_other': list(config[group].keys())[0]}
                     raise exception.InvalidDomainConfig(reason=msg)
 
                 # Finally, we need to check if the group/option specified
                 # already exists in the original config - since if not, to keep
                 # with the semantics of an update, we need to fail with
                 # a DomainConfigNotFound
-                if not self.get_config_with_sensitive_info(domain_id,
-                                                           group, option):
+                if not self._get_config_with_sensitive_info(domain_id,
+                                                            group, option):
                     if option:
                         msg = _('option %(option)s in group %(group)s') % {
                             'group': group, 'option': option}
@@ -1131,6 +1156,7 @@ class DomainConfigManager(manager.Manager):
         for new_option in sensitive:
             _update_or_create(domain_id, new_option, sensitive=True)
 
+        self.get_config_with_sensitive_info.invalidate(self, domain_id)
         return self.get_config(domain_id)
 
     def delete_config(self, domain_id, group=None, option=None):
@@ -1154,7 +1180,7 @@ class DomainConfigManager(manager.Manager):
         if group:
             # As this is a partial delete, then make sure the items requested
             # are valid and exist in the current config
-            current_config = self.get_config_with_sensitive_info(domain_id)
+            current_config = self._get_config_with_sensitive_info(domain_id)
             # Raise an exception if the group/options specified don't exist in
             # the current config so that the delete method provides the
             # correct error semantics.
@@ -1171,14 +1197,14 @@ class DomainConfigManager(manager.Manager):
 
         self.delete_config_options(domain_id, group, option)
         self.delete_config_options(domain_id, group, option, sensitive=True)
+        self.get_config_with_sensitive_info.invalidate(self, domain_id)
 
-    def get_config_with_sensitive_info(self, domain_id, group=None,
-                                       option=None):
-        """Get config for a domain with sensitive info included.
+    def _get_config_with_sensitive_info(self, domain_id, group=None,
+                                        option=None):
+        """Get config for a domain/group/option with sensitive info included.
 
-        This method is not exposed via the public API, but is used by the
-        identity manager to initialize a domain with the fully formed config
-        options.
+        This is only used by the methods within this class, which may need to
+        check individual groups or options.
 
         """
         whitelisted = self.list_config_options(domain_id, group, option)
@@ -1233,6 +1259,17 @@ class DomainConfigManager(manager.Manager):
 
         return self._list_to_config(whitelisted, sensitive)
 
+    @MEMOIZE_CONFIG
+    def get_config_with_sensitive_info(self, domain_id):
+        """Get config for a domain with sensitive info included.
+
+        This method is not exposed via the public API, but is used by the
+        identity manager to initialize a domain with the fully formed config
+        options.
+
+        """
+        return self._get_config_with_sensitive_info(domain_id)
+
 
 @six.add_metaclass(abc.ABCMeta)
 class DomainConfigDriver(object):
diff --git a/keystone-moon/keystone/resource/schema.py b/keystone-moon/keystone/resource/schema.py
index 0fd59e3f..e26a9c4a 100644
--- a/keystone-moon/keystone/resource/schema.py
+++ b/keystone-moon/keystone/resource/schema.py
@@ -21,6 +21,7 @@ _project_properties = {
     # implementation.
     'domain_id': parameter_types.id_string,
     'enabled': parameter_types.boolean,
+    'is_domain': parameter_types.boolean,
     'parent_id': validation.nullable(parameter_types.id_string),
     'name': {
         'type': 'string',