diff options
Diffstat (limited to 'keystone-moon/keystone/common')
55 files changed, 1091 insertions, 715 deletions
diff --git a/keystone-moon/keystone/common/authorization.py b/keystone-moon/keystone/common/authorization.py index 5cb1e630..8db618df 100644 --- a/keystone-moon/keystone/common/authorization.py +++ b/keystone-moon/keystone/common/authorization.py @@ -59,6 +59,7 @@ def token_to_auth_context(token): auth_context['project_id'] = token.project_id elif token.domain_scoped: auth_context['domain_id'] = token.domain_id + auth_context['domain_name'] = token.domain_name else: LOG.debug('RBAC: Proceeding without project or domain scope') diff --git a/keystone-moon/keystone/common/base64utils.py b/keystone-moon/keystone/common/base64utils.py index 1a636f9b..d19eade7 100644 --- a/keystone-moon/keystone/common/base64utils.py +++ b/keystone-moon/keystone/common/base64utils.py @@ -57,8 +57,13 @@ base64url_non_alphabet_re = re.compile(r'[^A-Za-z0-9---_=]+') _strip_formatting_re = re.compile(r'\s+') -_base64_to_base64url_trans = string.maketrans('+/', '-_') -_base64url_to_base64_trans = string.maketrans('-_', '+/') +if six.PY2: + str_ = string +else: + str_ = str + +_base64_to_base64url_trans = str_.maketrans('+/', '-_') +_base64url_to_base64_trans = str_.maketrans('-_', '+/') def _check_padding_length(pad): diff --git a/keystone-moon/keystone/common/cache/_memcache_pool.py b/keystone-moon/keystone/common/cache/_memcache_pool.py index b15332db..2bfcc3bb 100644 --- a/keystone-moon/keystone/common/cache/_memcache_pool.py +++ b/keystone-moon/keystone/common/cache/_memcache_pool.py @@ -27,7 +27,7 @@ import time import memcache from oslo_log import log -from six.moves import queue +from six.moves import queue, zip from keystone import exception from keystone.i18n import _ @@ -35,11 +35,22 @@ from keystone.i18n import _ LOG = log.getLogger(__name__) -# This 'class' is taken from http://stackoverflow.com/a/22520633/238308 -# Don't inherit client from threading.local so that we can reuse clients in -# different threads -_MemcacheClient = type('_MemcacheClient', (object,), - dict(memcache.Client.__dict__)) + +class _MemcacheClient(memcache.Client): + """Thread global memcache client + + As client is inherited from threading.local we have to restore object + methods overloaded by threading.local so we can reuse clients in + different threads + """ + __delattr__ = object.__delattr__ + __getattribute__ = object.__getattribute__ + __new__ = object.__new__ + __setattr__ = object.__setattr__ + + def __del__(self): + pass + _PoolItem = collections.namedtuple('_PoolItem', ['ttl', 'connection']) diff --git a/keystone-moon/keystone/common/cache/backends/mongo.py b/keystone-moon/keystone/common/cache/backends/mongo.py index b5de9bc4..cb5ad833 100644 --- a/keystone-moon/keystone/common/cache/backends/mongo.py +++ b/keystone-moon/keystone/common/cache/backends/mongo.py @@ -360,8 +360,12 @@ class MongoApi(object): self._assign_data_mainpulator() if self.read_preference: - self.read_preference = pymongo.read_preferences.mongos_enum( - self.read_preference) + # pymongo 3.0 renamed mongos_enum to read_pref_mode_from_name + f = getattr(pymongo.read_preferences, + 'read_pref_mode_from_name', None) + if not f: + f = pymongo.read_preferences.mongos_enum + self.read_preference = f(self.read_preference) coll.read_preference = self.read_preference if self.w > -1: coll.write_concern['w'] = self.w @@ -395,7 +399,7 @@ class MongoApi(object): Refer to MongoDB documentation around TTL index for further details. """ indexes = collection.index_information() - for indx_name, index_data in six.iteritems(indexes): + for indx_name, index_data in indexes.items(): if all(k in index_data for k in ('key', 'expireAfterSeconds')): existing_value = index_data['expireAfterSeconds'] fld_present = 'doc_date' in index_data['key'][0] @@ -447,7 +451,7 @@ class MongoApi(object): doc_date = self._get_doc_date() insert_refs = [] update_refs = [] - existing_docs = self._get_results_as_dict(mapping.keys()) + existing_docs = self._get_results_as_dict(list(mapping.keys())) for key, value in mapping.items(): ref = self._get_cache_entry(key, value.payload, value.metadata, doc_date) @@ -532,7 +536,7 @@ class BaseTransform(AbstractManipulator): def transform_incoming(self, son, collection): """Used while saving data to MongoDB.""" - for (key, value) in son.items(): + for (key, value) in list(son.items()): if isinstance(value, api.CachedValue): son[key] = value.payload # key is 'value' field here son['meta'] = value.metadata @@ -549,7 +553,7 @@ class BaseTransform(AbstractManipulator): ('_id', 'value', 'meta', 'doc_date')): payload = son.pop('value', None) metadata = son.pop('meta', None) - for (key, value) in son.items(): + for (key, value) in list(son.items()): if isinstance(value, dict): son[key] = self.transform_outgoing(value, collection) if metadata is not None: diff --git a/keystone-moon/keystone/common/clean.py b/keystone-moon/keystone/common/clean.py new file mode 100644 index 00000000..38564e0b --- /dev/null +++ b/keystone-moon/keystone/common/clean.py @@ -0,0 +1,87 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six + +from keystone import exception +from keystone.i18n import _ + + +def check_length(property_name, value, min_length=1, max_length=64): + if len(value) < min_length: + if min_length == 1: + msg = _("%s cannot be empty.") % property_name + else: + msg = (_("%(property_name)s cannot be less than " + "%(min_length)s characters.") % dict( + property_name=property_name, min_length=min_length)) + raise exception.ValidationError(msg) + if len(value) > max_length: + msg = (_("%(property_name)s should not be greater than " + "%(max_length)s characters.") % dict( + property_name=property_name, max_length=max_length)) + + raise exception.ValidationError(msg) + + +def check_type(property_name, value, expected_type, display_expected_type): + if not isinstance(value, expected_type): + msg = (_("%(property_name)s is not a " + "%(display_expected_type)s") % dict( + property_name=property_name, + display_expected_type=display_expected_type)) + raise exception.ValidationError(msg) + + +def check_enabled(property_name, enabled): + # Allow int and it's subclass bool + check_type('%s enabled' % property_name, enabled, int, 'boolean') + return bool(enabled) + + +def check_name(property_name, name, min_length=1, max_length=64): + check_type('%s name' % property_name, name, six.string_types, + 'str or unicode') + name = name.strip() + check_length('%s name' % property_name, name, + min_length=min_length, max_length=max_length) + return name + + +def domain_name(name): + return check_name('Domain', name) + + +def domain_enabled(enabled): + return check_enabled('Domain', enabled) + + +def project_name(name): + return check_name('Project', name) + + +def project_enabled(enabled): + return check_enabled('Project', enabled) + + +def user_name(name): + return check_name('User', name, max_length=255) + + +def user_enabled(enabled): + return check_enabled('User', enabled) + + +def group_name(name): + return check_name('Group', name) diff --git a/keystone-moon/keystone/common/config.py b/keystone-moon/keystone/common/config.py index bcaedeef..6cc848b4 100644 --- a/keystone-moon/keystone/common/config.py +++ b/keystone-moon/keystone/common/config.py @@ -14,6 +14,7 @@ from oslo_config import cfg import oslo_messaging +import passlib.utils _DEFAULT_AUTH_METHODS = ['external', 'password', 'token', 'oauth1'] @@ -32,14 +33,6 @@ FILE_OPTIONS = { 'AdminTokenAuthMiddleware from your paste ' 'application pipelines (for example, in ' 'keystone-paste.ini).'), - cfg.IntOpt('compute_port', default=8774, - help='(Deprecated) The port which the OpenStack Compute ' - 'service listens on. This option was only used for ' - 'string replacement in the templated catalog backend. ' - 'Templated catalogs should replace the ' - '"$(compute_port)s" substitution with the static port ' - 'of the compute service. As of Juno, this option is ' - 'deprecated and will be removed in the L release.'), cfg.StrOpt('public_endpoint', help='The base public endpoint URL for Keystone that is ' 'advertised to clients (NOTE: this does NOT affect ' @@ -81,7 +74,13 @@ FILE_OPTIONS = { help='This is the role name used in combination with the ' 'member_role_id option; see that option for more ' 'detail.'), - cfg.IntOpt('crypt_strength', default=40000, + # NOTE(lbragstad/morganfainberg): This value of 10k was + # measured as having an approximate 30% clock-time savings + # over the old default of 40k. The passlib default is not + # static and grows over time to constantly approximate ~300ms + # of CPU time to hash; this was considered too high. This + # value still exceeds the glibc default of 5k. + cfg.IntOpt('crypt_strength', default=10000, min=1000, max=100000, help='The value passed as the keyword "rounds" to ' 'passlib\'s encrypt method.'), cfg.IntOpt('list_limit', @@ -149,9 +148,10 @@ FILE_OPTIONS = { 'identity configuration files if ' 'domain_specific_drivers_enabled is set to true.'), cfg.StrOpt('driver', - default=('keystone.identity.backends' - '.sql.Identity'), - help='Identity backend driver.'), + default='sql', + help='Entrypoint for the identity backend driver in the ' + 'keystone.identity namespace. Supplied drivers are ' + 'ldap and sql.'), cfg.BoolOpt('caching', default=True, help='Toggle for identity caching. This has no ' 'effect unless global caching is enabled.'), @@ -160,6 +160,7 @@ FILE_OPTIONS = { 'no effect unless global and identity caching are ' 'enabled.'), cfg.IntOpt('max_password_length', default=4096, + max=passlib.utils.MAX_PASSWORD_SIZE, help='Maximum supported length for user passwords; ' 'decrease to improve performance.'), cfg.IntOpt('list_limit', @@ -168,15 +169,16 @@ FILE_OPTIONS = { ], 'identity_mapping': [ cfg.StrOpt('driver', - default=('keystone.identity.mapping_backends' - '.sql.Mapping'), - help='Keystone Identity Mapping backend driver.'), + default='sql', + help='Entrypoint for the identity mapping backend driver ' + 'in the keystone.identity.id_mapping namespace.'), cfg.StrOpt('generator', - default=('keystone.identity.id_generators' - '.sha256.Generator'), - help='Public ID generator for user and group entities. ' - 'The Keystone identity mapper only supports ' - 'generators that produce no more than 64 characters.'), + default='sha256', + help='Entrypoint for the public ID generator for user and ' + 'group entities in the keystone.identity.id_generator ' + 'namespace. The Keystone identity mapper only ' + 'supports generators that produce no more than 64 ' + 'characters.'), cfg.BoolOpt('backward_compatible_ids', default=True, help='The format of user and group IDs changed ' @@ -209,8 +211,9 @@ FILE_OPTIONS = { cfg.IntOpt('max_redelegation_count', default=3, help='Maximum depth of trust redelegation.'), cfg.StrOpt('driver', - default='keystone.trust.backends.sql.Trust', - help='Trust backend driver.')], + default='sql', + help='Entrypoint for the trust backend driver in the ' + 'keystone.trust namespace.')], 'os_inherit': [ cfg.BoolOpt('enabled', default=False, help='role-assignment inheritance to projects from ' @@ -245,14 +248,17 @@ FILE_OPTIONS = { help='Amount of time a token should remain valid ' '(in seconds).'), cfg.StrOpt('provider', - default='keystone.token.providers.uuid.Provider', + default='uuid', help='Controls the token construction, validation, and ' - 'revocation operations. Core providers are ' - '"keystone.token.providers.[fernet|pkiz|pki|uuid].' - 'Provider".'), + 'revocation operations. Entrypoint in the ' + 'keystone.token.provider namespace. Core providers ' + 'are [fernet|pkiz|pki|uuid].'), cfg.StrOpt('driver', - default='keystone.token.persistence.backends.sql.Token', - help='Token persistence backend driver.'), + default='sql', + help='Entrypoint for the token persistence backend driver ' + 'in the keystone.token.persistence namespace. ' + 'Supplied drivers are kvs, memcache, memcache_pool, ' + 'and sql.'), cfg.BoolOpt('caching', default=True, help='Toggle for token system caching. This has no ' 'effect unless global caching is enabled.'), @@ -282,9 +288,10 @@ FILE_OPTIONS = { ], 'revoke': [ cfg.StrOpt('driver', - default='keystone.contrib.revoke.backends.sql.Revoke', - help='An implementation of the backend for persisting ' - 'revocation events.'), + default='sql', + help='Entrypoint for an implementation of the backend for ' + 'persisting revocation events in the keystone.revoke ' + 'namespace. Supplied drivers are kvs and sql.'), cfg.IntOpt('expiration_buffer', default=1800, help='This value (calculated in seconds) is added to token ' 'expiration before a revocation event may be removed ' @@ -326,7 +333,7 @@ FILE_OPTIONS = { 'deployments. Small workloads (single process) ' 'like devstack can use the dogpile.cache.memory ' 'backend.'), - cfg.MultiStrOpt('backend_argument', default=[], + cfg.MultiStrOpt('backend_argument', default=[], secret=True, help='Arguments supplied to the backend module. ' 'Specify this option once per argument to be ' 'passed to the dogpile.cache backend. Example ' @@ -379,7 +386,7 @@ FILE_OPTIONS = { cfg.StrOpt('ca_key', default='/etc/keystone/ssl/private/cakey.pem', help='Path of the CA key file for SSL.'), - cfg.IntOpt('key_size', default=1024, + cfg.IntOpt('key_size', default=1024, min=1024, help='SSL key length (in bits) (auto generated ' 'certificate).'), cfg.IntOpt('valid_days', default=3650, @@ -406,7 +413,7 @@ FILE_OPTIONS = { cfg.StrOpt('ca_key', default='/etc/keystone/ssl/private/cakey.pem', help='Path of the CA key for token signing.'), - cfg.IntOpt('key_size', default=2048, + cfg.IntOpt('key_size', default=2048, min=1024, help='Key size (in bits) for token signing cert ' '(auto generated certificate).'), cfg.IntOpt('valid_days', default=3650, @@ -419,17 +426,20 @@ FILE_OPTIONS = { 'token signing.'), ], 'assignment': [ - # assignment has no default for backward compatibility reasons. - # If assignment driver is not specified, the identity driver chooses - # the backend cfg.StrOpt('driver', - help='Assignment backend driver.'), + help='Entrypoint for the assignment backend driver in the ' + 'keystone.assignment namespace. Supplied drivers are ' + 'ldap and sql. If an assignment driver is not ' + 'specified, the identity driver will choose the ' + 'assignment driver.'), ], 'resource': [ cfg.StrOpt('driver', - help='Resource backend driver. If a resource driver is ' - 'not specified, the assignment driver will choose ' - 'the resource driver.'), + help='Entrypoint for the resource backend driver in the ' + 'keystone.resource namespace. Supplied drivers are ' + 'ldap and sql. If a resource driver is not specified, ' + 'the assignment driver will choose the resource ' + 'driver.'), cfg.BoolOpt('caching', default=True, deprecated_opts=[cfg.DeprecatedOpt('caching', group='assignment')], @@ -448,16 +458,25 @@ FILE_OPTIONS = { ], 'domain_config': [ cfg.StrOpt('driver', - default='keystone.resource.config_backends.sql.' - 'DomainConfig', - help='Domain config backend driver.'), + default='sql', + help='Entrypoint for the domain config backend driver in ' + 'the keystone.resource.domain_config namespace.'), + cfg.BoolOpt('caching', default=True, + help='Toggle for domain config caching. This has no ' + 'effect unless global caching is enabled.'), + cfg.IntOpt('cache_time', default=300, + help='TTL (in seconds) to cache domain config data. This ' + 'has no effect unless domain config caching is ' + 'enabled.'), ], 'role': [ # The role driver has no default for backward compatibility reasons. # If role driver is not specified, the assignment driver chooses # the backend cfg.StrOpt('driver', - help='Role backend driver.'), + help='Entrypoint for the role backend driver in the ' + 'keystone.role namespace. Supplied drivers are ldap ' + 'and sql.'), cfg.BoolOpt('caching', default=True, help='Toggle for role caching. This has no effect ' 'unless global caching is enabled.'), @@ -470,14 +489,15 @@ FILE_OPTIONS = { ], 'credential': [ cfg.StrOpt('driver', - default=('keystone.credential.backends' - '.sql.Credential'), - help='Credential backend driver.'), + default='sql', + help='Entrypoint for the credential backend driver in the ' + 'keystone.credential namespace.'), ], 'oauth1': [ cfg.StrOpt('driver', - default='keystone.contrib.oauth1.backends.sql.OAuth1', - help='Credential backend driver.'), + default='sql', + help='Entrypoint for hte OAuth backend driver in the ' + 'keystone.oauth1 namespace.'), cfg.IntOpt('request_token_duration', default=28800, help='Duration (in seconds) for the OAuth Request Token.'), cfg.IntOpt('access_token_duration', default=86400, @@ -485,9 +505,9 @@ FILE_OPTIONS = { ], 'federation': [ cfg.StrOpt('driver', - default='keystone.contrib.federation.' - 'backends.sql.Federation', - help='Federation backend driver.'), + default='sql', + help='Entrypoint for the federation backend driver in the ' + 'keystone.federation namespace.'), cfg.StrOpt('assertion_prefix', default='', help='Value to be used when filtering assertion parameters ' 'from the environment.'), @@ -502,9 +522,7 @@ FILE_OPTIONS = { 'an admin will not be able to create a domain with ' 'this name or update an existing domain to this ' 'name. You are not advised to change this value ' - 'unless you really have to. Changing this option ' - 'to empty string or None will not have any impact and ' - 'default name will be used.'), + 'unless you really have to.'), cfg.MultiStrOpt('trusted_dashboard', default=[], help='A list of trusted dashboard hosts. Before ' 'accepting a Single Sign-On request to return a ' @@ -519,26 +537,31 @@ FILE_OPTIONS = { ], 'policy': [ cfg.StrOpt('driver', - default='keystone.policy.backends.sql.Policy', - help='Policy backend driver.'), + default='sql', + help='Entrypoint for the policy backend driver in the ' + 'keystone.policy namespace. Supplied drivers are ' + 'rules and sql.'), cfg.IntOpt('list_limit', help='Maximum number of entities that will be returned ' 'in a policy collection.'), ], 'endpoint_filter': [ cfg.StrOpt('driver', - default='keystone.contrib.endpoint_filter.backends' - '.sql.EndpointFilter', - help='Endpoint Filter backend driver'), + default='sql', + help='Entrypoint for the endpoint filter backend driver in ' + 'the keystone.endpoint_filter namespace.'), cfg.BoolOpt('return_all_endpoints_if_no_filter', default=True, help='Toggle to return all active endpoints if no filter ' 'exists.'), ], 'endpoint_policy': [ + cfg.BoolOpt('enabled', + default=True, + help='Enable endpoint_policy functionality.'), cfg.StrOpt('driver', - default='keystone.contrib.endpoint_policy.backends' - '.sql.EndpointPolicy', - help='Endpoint policy backend driver'), + default='sql', + help='Entrypoint for the endpoint policy backend driver in ' + 'the keystone.endpoint_policy namespace.'), ], 'ldap': [ cfg.StrOpt('url', default='ldap://localhost', @@ -561,18 +584,19 @@ FILE_OPTIONS = { 'Only enable this option if your LDAP server ' 'supports subtree deletion.'), cfg.StrOpt('query_scope', default='one', - help='The LDAP scope for queries, this can be either ' - '"one" (onelevel/singleLevel) or "sub" ' - '(subtree/wholeSubtree).'), + choices=['one', 'sub'], + help='The LDAP scope for queries, "one" represents ' + 'oneLevel/singleLevel and "sub" represents ' + 'subtree/wholeSubtree options.'), cfg.IntOpt('page_size', default=0, help='Maximum results per page; a value of zero ("0") ' 'disables paging.'), cfg.StrOpt('alias_dereferencing', default='default', - help='The LDAP dereferencing option for queries. This ' - 'can be either "never", "searching", "always", ' - '"finding" or "default". The "default" option falls ' - 'back to using default dereferencing configured by ' - 'your ldap.conf.'), + choices=['never', 'searching', 'always', 'finding', + 'default'], + help='The LDAP dereferencing option for queries. The ' + '"default" option falls back to using default ' + 'dereferencing configured by your ldap.conf.'), cfg.IntOpt('debug_level', help='Sets the LDAP debugging level for LDAP calls. ' 'A value of 0 means that debugging is not enabled. ' @@ -582,7 +606,8 @@ FILE_OPTIONS = { help='Override the system\'s default referral chasing ' 'behavior for queries.'), cfg.StrOpt('user_tree_dn', - help='Search base for users.'), + help='Search base for users. ' + 'Defaults to the suffix value.'), cfg.StrOpt('user_filter', help='LDAP search filter for users.'), cfg.StrOpt('user_objectclass', default='inetOrgPerson', @@ -622,7 +647,7 @@ FILE_OPTIONS = { 'the typical value is "512". This is typically used ' 'when "user_enabled_attribute = userAccountControl".'), cfg.ListOpt('user_attribute_ignore', - default=['default_project_id', 'tenants'], + default=['default_project_id'], help='List of attributes stripped off the user on ' 'update.'), cfg.StrOpt('user_default_project_id_attribute', @@ -653,61 +678,76 @@ FILE_OPTIONS = { cfg.StrOpt('project_tree_dn', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_tree_dn', group='ldap')], - help='Search base for projects'), + deprecated_for_removal=True, + help='Search base for projects. ' + 'Defaults to the suffix value.'), cfg.StrOpt('project_filter', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_filter', group='ldap')], + deprecated_for_removal=True, help='LDAP search filter for projects.'), cfg.StrOpt('project_objectclass', default='groupOfNames', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_objectclass', group='ldap')], + deprecated_for_removal=True, help='LDAP objectclass for projects.'), cfg.StrOpt('project_id_attribute', default='cn', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_id_attribute', group='ldap')], + deprecated_for_removal=True, help='LDAP attribute mapped to project id.'), cfg.StrOpt('project_member_attribute', default='member', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_member_attribute', group='ldap')], + deprecated_for_removal=True, help='LDAP attribute mapped to project membership for ' 'user.'), cfg.StrOpt('project_name_attribute', default='ou', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_name_attribute', group='ldap')], + deprecated_for_removal=True, help='LDAP attribute mapped to project name.'), cfg.StrOpt('project_desc_attribute', default='description', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_desc_attribute', group='ldap')], + deprecated_for_removal=True, help='LDAP attribute mapped to project description.'), cfg.StrOpt('project_enabled_attribute', default='enabled', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_enabled_attribute', group='ldap')], + deprecated_for_removal=True, help='LDAP attribute mapped to project enabled.'), cfg.StrOpt('project_domain_id_attribute', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_domain_id_attribute', group='ldap')], + deprecated_for_removal=True, default='businessCategory', help='LDAP attribute mapped to project domain_id.'), cfg.ListOpt('project_attribute_ignore', default=[], deprecated_opts=[cfg.DeprecatedOpt( 'tenant_attribute_ignore', group='ldap')], + deprecated_for_removal=True, help='List of attributes stripped off the project on ' 'update.'), cfg.BoolOpt('project_allow_create', default=True, deprecated_opts=[cfg.DeprecatedOpt( 'tenant_allow_create', group='ldap')], + deprecated_for_removal=True, help='Allow project creation in LDAP backend.'), cfg.BoolOpt('project_allow_update', default=True, deprecated_opts=[cfg.DeprecatedOpt( 'tenant_allow_update', group='ldap')], + deprecated_for_removal=True, help='Allow project update in LDAP backend.'), cfg.BoolOpt('project_allow_delete', default=True, deprecated_opts=[cfg.DeprecatedOpt( 'tenant_allow_delete', group='ldap')], + deprecated_for_removal=True, help='Allow project deletion in LDAP backend.'), cfg.BoolOpt('project_enabled_emulation', default=False, deprecated_opts=[cfg.DeprecatedOpt( 'tenant_enabled_emulation', group='ldap')], + deprecated_for_removal=True, help='If true, Keystone uses an alternative method to ' 'determine if a project is enabled or not by ' 'checking if they are a member of the ' @@ -715,11 +755,13 @@ FILE_OPTIONS = { cfg.StrOpt('project_enabled_emulation_dn', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_enabled_emulation_dn', group='ldap')], + deprecated_for_removal=True, help='DN of the group entry to hold enabled projects when ' 'using enabled emulation.'), cfg.ListOpt('project_additional_attribute_mapping', deprecated_opts=[cfg.DeprecatedOpt( 'tenant_additional_attribute_mapping', group='ldap')], + deprecated_for_removal=True, default=[], help='Additional attribute mappings for projects. ' 'Attribute mapping format is ' @@ -728,27 +770,39 @@ FILE_OPTIONS = { 'Identity API attribute.'), cfg.StrOpt('role_tree_dn', - help='Search base for roles.'), + deprecated_for_removal=True, + help='Search base for roles. ' + 'Defaults to the suffix value.'), cfg.StrOpt('role_filter', + deprecated_for_removal=True, help='LDAP search filter for roles.'), cfg.StrOpt('role_objectclass', default='organizationalRole', + deprecated_for_removal=True, help='LDAP objectclass for roles.'), cfg.StrOpt('role_id_attribute', default='cn', + deprecated_for_removal=True, help='LDAP attribute mapped to role id.'), cfg.StrOpt('role_name_attribute', default='ou', + deprecated_for_removal=True, help='LDAP attribute mapped to role name.'), cfg.StrOpt('role_member_attribute', default='roleOccupant', + deprecated_for_removal=True, help='LDAP attribute mapped to role membership.'), cfg.ListOpt('role_attribute_ignore', default=[], + deprecated_for_removal=True, help='List of attributes stripped off the role on ' 'update.'), cfg.BoolOpt('role_allow_create', default=True, + deprecated_for_removal=True, help='Allow role creation in LDAP backend.'), cfg.BoolOpt('role_allow_update', default=True, + deprecated_for_removal=True, help='Allow role update in LDAP backend.'), cfg.BoolOpt('role_allow_delete', default=True, + deprecated_for_removal=True, help='Allow role deletion in LDAP backend.'), cfg.ListOpt('role_additional_attribute_mapping', + deprecated_for_removal=True, default=[], help='Additional attribute mappings for roles. Attribute ' 'mapping format is <ldap_attr>:<user_attr>, where ' @@ -756,7 +810,8 @@ FILE_OPTIONS = { 'user_attr is the Identity API attribute.'), cfg.StrOpt('group_tree_dn', - help='Search base for groups.'), + help='Search base for groups. ' + 'Defaults to the suffix value.'), cfg.StrOpt('group_filter', help='LDAP search filter for groups.'), cfg.StrOpt('group_objectclass', default='groupOfNames', @@ -794,8 +849,9 @@ FILE_OPTIONS = { cfg.BoolOpt('use_tls', default=False, help='Enable TLS for communicating with LDAP servers.'), cfg.StrOpt('tls_req_cert', default='demand', - help='Valid options for tls_req_cert are demand, never, ' - 'and allow.'), + choices=['demand', 'never', 'allow'], + help='Specifies what checks to perform on client ' + 'certificates in an incoming TLS session.'), cfg.BoolOpt('use_pool', default=False, help='Enable LDAP connection pooling.'), cfg.IntOpt('pool_size', default=10, @@ -821,20 +877,22 @@ FILE_OPTIONS = { ], 'auth': [ cfg.ListOpt('methods', default=_DEFAULT_AUTH_METHODS, - help='Default auth methods.'), + help='Allowed authentication methods.'), cfg.StrOpt('password', - default='keystone.auth.plugins.password.Password', - help='The password auth plugin module.'), + help='Entrypoint for the password auth plugin module in ' + 'the keystone.auth.password namespace.'), cfg.StrOpt('token', - default='keystone.auth.plugins.token.Token', - help='The token auth plugin module.'), + help='Entrypoint for the token auth plugin module in the ' + 'keystone.auth.token namespace.'), # deals with REMOTE_USER authentication cfg.StrOpt('external', - default='keystone.auth.plugins.external.DefaultDomain', - help='The external (REMOTE_USER) auth plugin module.'), + help='Entrypoint for the external (REMOTE_USER) auth ' + 'plugin module in the keystone.auth.external ' + 'namespace. Supplied drivers are DefaultDomain and ' + 'Domain. The default driver is DefaultDomain.'), cfg.StrOpt('oauth1', - default='keystone.auth.plugins.oauth1.OAuth', - help='The oAuth1.0 auth plugin module.'), + help='Entrypoint for the oAuth1.0 auth plugin module in ' + 'the keystone.auth.oauth1 namespace.'), ], 'paste_deploy': [ cfg.StrOpt('config_file', default='keystone-paste.ini', @@ -880,8 +938,10 @@ FILE_OPTIONS = { help='Catalog template file name for use with the ' 'template catalog backend.'), cfg.StrOpt('driver', - default='keystone.catalog.backends.sql.Catalog', - help='Catalog backend driver.'), + default='sql', + help='Entrypoint for the catalog backend driver in the ' + 'keystone.catalog namespace. Supplied drivers are ' + 'kvs, sql, templated, and endpoint_filter.sql'), cfg.BoolOpt('caching', default=True, help='Toggle for catalog caching. This has no ' 'effect unless global caching is enabled.'), @@ -963,25 +1023,33 @@ FILE_OPTIONS = { cfg.StrOpt('idp_contact_telephone', help='Telephone number of contact person.'), cfg.StrOpt('idp_contact_type', default='other', - help='Contact type. Allowed values are: ' - 'technical, support, administrative ' - 'billing, and other'), + choices=['technical', 'support', 'administrative', + 'billing', 'other'], + help='The contact type describing the main point of ' + 'contact for the identity provider.'), cfg.StrOpt('idp_metadata_path', default='/etc/keystone/saml2_idp_metadata.xml', help='Path to the Identity Provider Metadata file. ' 'This file should be generated with the ' 'keystone-manage saml_idp_metadata command.'), + cfg.StrOpt('relay_state_prefix', + default='ss:mem:', + help='The prefix to use for the RelayState SAML ' + 'attribute, used when generating ECP wrapped ' + 'assertions.'), ], 'eventlet_server': [ cfg.IntOpt('public_workers', deprecated_name='public_workers', deprecated_group='DEFAULT', + deprecated_for_removal=True, help='The number of worker processes to serve the public ' 'eventlet application. Defaults to number of CPUs ' '(minimum of 2).'), cfg.IntOpt('admin_workers', deprecated_name='admin_workers', deprecated_group='DEFAULT', + deprecated_for_removal=True, help='The number of worker processes to serve the admin ' 'eventlet application. Defaults to number of CPUs ' '(minimum of 2).'), @@ -991,10 +1059,13 @@ FILE_OPTIONS = { group='DEFAULT'), cfg.DeprecatedOpt('public_bind_host', group='DEFAULT'), ], + deprecated_for_removal=True, help='The IP address of the network interface for the ' 'public service to listen on.'), - cfg.IntOpt('public_port', default=5000, deprecated_name='public_port', + cfg.IntOpt('public_port', default=5000, min=1, max=65535, + deprecated_name='public_port', deprecated_group='DEFAULT', + deprecated_for_removal=True, help='The port number which the public service listens ' 'on.'), cfg.StrOpt('admin_bind_host', @@ -1003,15 +1074,28 @@ FILE_OPTIONS = { group='DEFAULT'), cfg.DeprecatedOpt('admin_bind_host', group='DEFAULT')], + deprecated_for_removal=True, help='The IP address of the network interface for the ' 'admin service to listen on.'), - cfg.IntOpt('admin_port', default=35357, deprecated_name='admin_port', + cfg.IntOpt('admin_port', default=35357, min=1, max=65535, + deprecated_name='admin_port', deprecated_group='DEFAULT', + deprecated_for_removal=True, help='The port number which the admin service listens ' 'on.'), + cfg.BoolOpt('wsgi_keep_alive', default=True, + help="If set to false, disables keepalives on the server; " + "all connections will be closed after serving one " + "request."), + cfg.IntOpt('client_socket_timeout', default=900, + help="Timeout for socket operations on a client " + "connection. If an incoming connection is idle for " + "this number of seconds it will be closed. A value " + "of '0' means wait forever."), cfg.BoolOpt('tcp_keepalive', default=False, deprecated_name='tcp_keepalive', deprecated_group='DEFAULT', + deprecated_for_removal=True, help='Set this to true if you want to enable ' 'TCP_KEEPALIVE on server sockets, i.e. sockets used ' 'by the Keystone wsgi server for client ' @@ -1020,6 +1104,7 @@ FILE_OPTIONS = { default=600, deprecated_name='tcp_keepidle', deprecated_group='DEFAULT', + deprecated_for_removal=True, help='Sets the value of TCP_KEEPIDLE in seconds for each ' 'server socket. Only applies if tcp_keepalive is ' 'true.'), @@ -1027,11 +1112,13 @@ FILE_OPTIONS = { 'eventlet_server_ssl': [ cfg.BoolOpt('enable', default=False, deprecated_name='enable', deprecated_group='ssl', + deprecated_for_removal=True, help='Toggle for SSL support on the Keystone ' 'eventlet servers.'), cfg.StrOpt('certfile', default="/etc/keystone/ssl/certs/keystone.pem", deprecated_name='certfile', deprecated_group='ssl', + deprecated_for_removal=True, help='Path of the certfile for SSL. For non-production ' 'environments, you may be interested in using ' '`keystone-manage ssl_setup` to generate self-signed ' @@ -1039,13 +1126,16 @@ FILE_OPTIONS = { cfg.StrOpt('keyfile', default='/etc/keystone/ssl/private/keystonekey.pem', deprecated_name='keyfile', deprecated_group='ssl', + deprecated_for_removal=True, help='Path of the keyfile for SSL.'), cfg.StrOpt('ca_certs', default='/etc/keystone/ssl/certs/ca.pem', deprecated_name='ca_certs', deprecated_group='ssl', + deprecated_for_removal=True, help='Path of the CA cert file for SSL.'), cfg.BoolOpt('cert_required', default=False, deprecated_name='cert_required', deprecated_group='ssl', + deprecated_for_removal=True, help='Require client certificate.'), ], } @@ -1080,7 +1170,7 @@ def configure(conf=None): cfg.StrOpt('pydev-debug-host', help='Host to connect to for remote debugger.')) conf.register_cli_opt( - cfg.IntOpt('pydev-debug-port', + cfg.IntOpt('pydev-debug-port', min=1, max=65535, help='Port to connect to for remote debugger.')) for section in FILE_OPTIONS: @@ -1115,4 +1205,4 @@ def list_opts(): :returns: a list of (group_name, opts) tuples """ - return FILE_OPTIONS.items() + return list(FILE_OPTIONS.items()) diff --git a/keystone-moon/keystone/common/controller.py b/keystone-moon/keystone/common/controller.py index bd26b7c4..bc7074ac 100644 --- a/keystone-moon/keystone/common/controller.py +++ b/keystone-moon/keystone/common/controller.py @@ -17,6 +17,7 @@ import uuid from oslo_config import cfg from oslo_log import log +from oslo_utils import strutils import six from keystone.common import authorization @@ -39,7 +40,7 @@ def v2_deprecated(f): This is a placeholder for the pending deprecation of v2. The implementation of this decorator can be replaced with:: - from keystone.openstack.common import versionutils + from oslo_log import versionutils v2_deprecated = versionutils.deprecated( @@ -52,9 +53,12 @@ def v2_deprecated(f): def _build_policy_check_credentials(self, action, context, kwargs): + kwargs_str = ', '.join(['%s=%s' % (k, kwargs[k]) for k in kwargs]) + kwargs_str = strutils.mask_password(kwargs_str) + LOG.debug('RBAC: Authorizing %(action)s(%(kwargs)s)', { 'action': action, - 'kwargs': ', '.join(['%s=%s' % (k, kwargs[k]) for k in kwargs])}) + 'kwargs': kwargs_str}) # see if auth context has already been created. If so use it. if ('environment' in context and @@ -219,7 +223,11 @@ class V2Controller(wsgi.Application): @staticmethod def filter_domain_id(ref): """Remove domain_id since v2 calls are not domain-aware.""" - ref.pop('domain_id', None) + if 'domain_id' in ref: + if ref['domain_id'] != CONF.identity.default_domain_id: + raise exception.Unauthorized( + _('Non-default domain is not supported')) + del ref['domain_id'] return ref @staticmethod @@ -239,6 +247,18 @@ class V2Controller(wsgi.Application): return ref @staticmethod + def filter_project_parent_id(ref): + """Remove parent_id since v2 calls are not hierarchy-aware.""" + ref.pop('parent_id', None) + return ref + + @staticmethod + def filter_is_domain(ref): + """Remove is_domain field since v2 calls are not domain-aware.""" + ref.pop('is_domain', None) + return ref + + @staticmethod def normalize_username_in_response(ref): """Adds username to outgoing user refs to match the v2 spec. @@ -266,9 +286,12 @@ class V2Controller(wsgi.Application): def v3_to_v2_user(ref): """Convert a user_ref from v3 to v2 compatible. - * v2.0 users are not domain aware, and should have domain_id removed - * v2.0 users expect the use of tenantId instead of default_project_id - * v2.0 users have a username attribute + - v2.0 users are not domain aware, and should have domain_id validated + to be the default domain, and then removed. + + - v2.0 users expect the use of tenantId instead of default_project_id. + + - v2.0 users have a username attribute. This method should only be applied to user_refs being returned from the v2.0 controller(s). @@ -304,6 +327,35 @@ class V2Controller(wsgi.Application): else: raise ValueError(_('Expected dict or list: %s') % type(ref)) + @staticmethod + def v3_to_v2_project(ref): + """Convert a project_ref from v3 to v2. + + * v2.0 projects are not domain aware, and should have domain_id removed + * v2.0 projects are not hierarchy aware, and should have parent_id + removed + + This method should only be applied to project_refs being returned from + the v2.0 controller(s). + + If ref is a list type, we will iterate through each element and do the + conversion. + """ + + def _filter_project_properties(ref): + """Run through the various filter methods.""" + V2Controller.filter_domain_id(ref) + V2Controller.filter_project_parent_id(ref) + V2Controller.filter_is_domain(ref) + return ref + + if isinstance(ref, dict): + return _filter_project_properties(ref) + elif isinstance(ref, list): + return [_filter_project_properties(x) for x in ref] + else: + raise ValueError(_('Expected dict or list: %s') % type(ref)) + def format_project_list(self, tenant_refs, **kwargs): """Format a v2 style project list, including marker/limits.""" marker = kwargs.get('marker') @@ -656,19 +708,7 @@ class V3Controller(wsgi.Application): if context['query_string'].get('domain_id') is not None: return context['query_string'].get('domain_id') - try: - token_ref = token_model.KeystoneToken( - token_id=context['token_id'], - token_data=self.token_provider_api.validate_token( - context['token_id'])) - except KeyError: - raise exception.ValidationError( - _('domain_id is required as part of entity')) - except (exception.TokenNotFound, - exception.UnsupportedTokenVersionException): - LOG.warning(_LW('Invalid token found while getting domain ID ' - 'for list request')) - raise exception.Unauthorized() + token_ref = utils.get_token_ref(context) if token_ref.domain_scoped: return token_ref.domain_id @@ -685,25 +725,7 @@ class V3Controller(wsgi.Application): being used. """ - # We could make this more efficient by loading the domain_id - # into the context in the wrapper function above (since - # this version of normalize_domain will only be called inside - # a v3 protected call). However, this optimization is probably not - # worth the duplication of state - try: - token_ref = token_model.KeystoneToken( - token_id=context['token_id'], - token_data=self.token_provider_api.validate_token( - context['token_id'])) - except KeyError: - # This might happen if we use the Admin token, for instance - raise exception.ValidationError( - _('A domain-scoped token must be used')) - except (exception.TokenNotFound, - exception.UnsupportedTokenVersionException): - LOG.warning(_LW('Invalid token found while getting domain ID ' - 'for list request')) - raise exception.Unauthorized() + token_ref = utils.get_token_ref(context) if token_ref.domain_scoped: return token_ref.domain_id diff --git a/keystone-moon/keystone/common/dependency.py b/keystone-moon/keystone/common/dependency.py index 14a68f19..e19f705f 100644 --- a/keystone-moon/keystone/common/dependency.py +++ b/keystone-moon/keystone/common/dependency.py @@ -15,9 +15,9 @@ """This module provides support for dependency injection. Providers are registered via the ``@provider()`` decorator, and dependencies on -them are registered with ``@requires()`` or ``@optional()``. Providers are -available to their consumers via an attribute. See the documentation for the -individual functions for more detail. +them are registered with ``@requires()``. Providers are available to their +consumers via an attribute. See the documentation for the individual functions +for more detail. See also: @@ -27,16 +27,12 @@ See also: import traceback -import six - from keystone.i18n import _ -from keystone import notifications _REGISTRY = {} _future_dependencies = {} -_future_optionals = {} _factories = {} @@ -94,44 +90,10 @@ def provider(name): """ def wrapper(cls): def wrapped(init): - def register_event_callbacks(self): - # NOTE(morganfainberg): A provider who has an implicit - # dependency on other providers may utilize the event callback - # mechanism to react to any changes in those providers. This is - # performed at the .provider() mechanism so that we can ensure - # that the callback is only ever called once and guaranteed - # to be on the properly configured and instantiated backend. - if not hasattr(self, 'event_callbacks'): - return - - if not isinstance(self.event_callbacks, dict): - msg = _('event_callbacks must be a dict') - raise ValueError(msg) - - for event in self.event_callbacks: - if not isinstance(self.event_callbacks[event], dict): - msg = _('event_callbacks[%s] must be a dict') % event - raise ValueError(msg) - for resource_type in self.event_callbacks[event]: - # Make sure we register the provider for each event it - # cares to call back. - callbacks = self.event_callbacks[event][resource_type] - if not callbacks: - continue - if not hasattr(callbacks, '__iter__'): - # ensure the callback information is a list - # allowing multiple callbacks to exist - callbacks = [callbacks] - notifications.register_event_callback(event, - resource_type, - callbacks) - def __wrapped_init__(self, *args, **kwargs): """Initialize the wrapped object and add it to the registry.""" init(self, *args, **kwargs) _set_provider(name, self) - register_event_callbacks(self) - resolve_future_dependencies(__provider_name=name) return __wrapped_init__ @@ -157,7 +119,6 @@ def _process_dependencies(obj): setattr(obj, dependency, get_provider(dependency)) process(obj, '_dependencies', _future_dependencies) - process(obj, '_optionals', _future_optionals) def requires(*dependencies): @@ -210,34 +171,6 @@ def requires(*dependencies): return wrapped -def optional(*dependencies): - """Similar to ``@requires()``, except that the dependencies are optional. - - If no provider is available, the attributes will be set to ``None``. - - """ - def wrapper(self, *args, **kwargs): - """Inject each dependency from the registry.""" - self.__wrapped_init__(*args, **kwargs) - _process_dependencies(self) - - def wrapped(cls): - """Note the optional dependencies on the object for later injection. - - The dependencies of the parent class are combined with that of the - child class to create a new set of dependencies. - - """ - existing_optionals = getattr(cls, '_optionals', set()) - cls._optionals = existing_optionals.union(dependencies) - if not hasattr(cls, '__wrapped_init__'): - cls.__wrapped_init__ = cls.__init__ - cls.__init__ = wrapper - return cls - - return wrapped - - def resolve_future_dependencies(__provider_name=None): """Forces injection of all dependencies. @@ -259,29 +192,16 @@ def resolve_future_dependencies(__provider_name=None): # A provider was registered, so take care of any objects depending on # it. targets = _future_dependencies.pop(__provider_name, []) - targets.extend(_future_optionals.pop(__provider_name, [])) for target in targets: setattr(target, __provider_name, get_provider(__provider_name)) return - # Resolve optional dependencies, sets the attribute to None if there's no - # provider registered. - for dependency, targets in six.iteritems(_future_optionals.copy()): - provider = get_provider(dependency, optional=GET_OPTIONAL) - if provider is None: - factory = _factories.get(dependency) - if factory: - provider = factory() - new_providers[dependency] = provider - for target in targets: - setattr(target, dependency, provider) - # Resolve future dependencies, raises UnresolvableDependencyException if # there's no provider registered. try: - for dependency, targets in six.iteritems(_future_dependencies.copy()): + for dependency, targets in _future_dependencies.copy().items(): if dependency not in _REGISTRY: # a Class was registered that could fulfill the dependency, but # it has not yet been initialized. @@ -308,4 +228,3 @@ def reset(): _REGISTRY.clear() _future_dependencies.clear() - _future_optionals.clear() diff --git a/keystone-moon/keystone/common/driver_hints.py b/keystone-moon/keystone/common/driver_hints.py index 0361e314..ff0a774c 100644 --- a/keystone-moon/keystone/common/driver_hints.py +++ b/keystone-moon/keystone/common/driver_hints.py @@ -30,6 +30,10 @@ class Hints(object): accessed publicly. Also it contains a dict called limit, which will indicate the amount of data we want to limit our listing to. + If the filter is discovered to never match, then `cannot_match` can be set + to indicate that there will not be any matches and the backend work can be + short-circuited. + Each filter term consists of: * ``name``: the name of the attribute being matched @@ -44,6 +48,7 @@ class Hints(object): def __init__(self): self.limit = None self.filters = list() + self.cannot_match = False def add_filter(self, name, value, comparator='equals', case_sensitive=False): diff --git a/keystone-moon/keystone/common/environment/__init__.py b/keystone-moon/keystone/common/environment/__init__.py index da1de890..3edf6b0b 100644 --- a/keystone-moon/keystone/common/environment/__init__.py +++ b/keystone-moon/keystone/common/environment/__init__.py @@ -17,6 +17,7 @@ import os from oslo_log import log + LOG = log.getLogger(__name__) @@ -93,7 +94,7 @@ def use_eventlet(monkeypatch_thread=None): def use_stdlib(): global httplib, subprocess - import httplib as _httplib + import six.moves.http_client as _httplib import subprocess as _subprocess httplib = _httplib diff --git a/keystone-moon/keystone/common/environment/eventlet_server.py b/keystone-moon/keystone/common/environment/eventlet_server.py index 639e074a..398952e1 100644 --- a/keystone-moon/keystone/common/environment/eventlet_server.py +++ b/keystone-moon/keystone/common/environment/eventlet_server.py @@ -25,12 +25,17 @@ import sys import eventlet import eventlet.wsgi import greenlet +from oslo_config import cfg from oslo_log import log from oslo_log import loggers +from oslo_service import service from keystone.i18n import _LE, _LI +CONF = cfg.CONF + + LOG = log.getLogger(__name__) # The size of a pool that is used to spawn a single green thread in which @@ -62,7 +67,7 @@ class EventletFilteringLogger(loggers.WritableLogger): self.logger.log(self.level, msg.rstrip()) -class Server(object): +class Server(service.ServiceBase): """Server class to manage multiple WSGI sockets and applications.""" def __init__(self, application, host=None, port=None, keepalive=False, @@ -173,7 +178,7 @@ class Server(object): The service interface is used by the launcher when receiving a SIGHUP. The service interface is defined in - keystone.openstack.common.service.Service. + oslo_service.service.Service. Keystone does not need to do anything here. """ @@ -182,10 +187,17 @@ class Server(object): def _run(self, application, socket): """Start a WSGI server with a new green thread pool.""" logger = log.getLogger('eventlet.wsgi.server') + + # NOTE(dolph): [eventlet_server] client_socket_timeout is required to + # be an integer in keystone.conf, but in order to make + # eventlet.wsgi.server() wait forever, we pass None instead of 0. + socket_timeout = CONF.eventlet_server.client_socket_timeout or None + try: - eventlet.wsgi.server(socket, application, - log=EventletFilteringLogger(logger), - debug=False) + eventlet.wsgi.server( + socket, application, log=EventletFilteringLogger(logger), + debug=False, keepalive=CONF.eventlet_server.wsgi_keep_alive, + socket_timeout=socket_timeout) except greenlet.GreenletExit: # Wait until all servers have completed running pass diff --git a/keystone-moon/keystone/common/json_home.py b/keystone-moon/keystone/common/json_home.py index 215d596a..c048a356 100644 --- a/keystone-moon/keystone/common/json_home.py +++ b/keystone-moon/keystone/common/json_home.py @@ -13,7 +13,8 @@ # under the License. -import six +from keystone import exception +from keystone.i18n import _ def build_v3_resource_relation(resource_name): @@ -62,14 +63,24 @@ class Status(object): STABLE = 'stable' @classmethod - def is_supported(cls, status): - return status in [cls.DEPRECATED, cls.EXPERIMENTAL, cls.STABLE] + def update_resource_data(cls, resource_data, status): + if status is cls.STABLE: + # We currently do not add a status if the resource is stable, the + # absence of the status property can be taken as meaning that the + # resource is stable. + return + if status is cls.DEPRECATED or status is cls.EXPERIMENTAL: + resource_data['hints'] = {'status': status} + return + + raise exception.Error(message=_( + 'Unexpected status requested for JSON Home response, %s') % status) def translate_urls(json_home, new_prefix): """Given a JSON Home document, sticks new_prefix on each of the urls.""" - for dummy_rel, resource in six.iteritems(json_home['resources']): + for dummy_rel, resource in json_home['resources'].items(): if 'href' in resource: resource['href'] = new_prefix + resource['href'] elif 'href-template' in resource: diff --git a/keystone-moon/keystone/common/kvs/backends/memcached.py b/keystone-moon/keystone/common/kvs/backends/memcached.py index db453143..f54c1a01 100644 --- a/keystone-moon/keystone/common/kvs/backends/memcached.py +++ b/keystone-moon/keystone/common/kvs/backends/memcached.py @@ -23,9 +23,9 @@ from dogpile.cache import api from dogpile.cache.backends import memcached from oslo_config import cfg from oslo_log import log +from six.moves import range from keystone.common.cache.backends import memcache_pool -from keystone.common import manager from keystone import exception from keystone.i18n import _ @@ -73,12 +73,13 @@ class MemcachedLock(object): client.delete(self.key) -class MemcachedBackend(manager.Manager): +class MemcachedBackend(object): """Pivot point to leverage the various dogpile.cache memcached backends. - To specify a specific dogpile.cache memcached driver, pass the argument - `memcached_driver` set to one of the provided memcached drivers (at this - time `memcached`, `bmemcached`, `pylibmc` are valid). + To specify a specific dogpile.cache memcached backend, pass the argument + `memcached_backend` set to one of the provided memcached backends (at this + time `memcached`, `bmemcached`, `pylibmc` and `pooled_memcached` are + valid). """ def __init__(self, arguments): self._key_mangler = None @@ -105,13 +106,19 @@ class MemcachedBackend(manager.Manager): else: if backend not in VALID_DOGPILE_BACKENDS: raise ValueError( - _('Backend `%(driver)s` is not a valid memcached ' - 'backend. Valid drivers: %(driver_list)s') % - {'driver': backend, - 'driver_list': ','.join(VALID_DOGPILE_BACKENDS.keys())}) + _('Backend `%(backend)s` is not a valid memcached ' + 'backend. Valid backends: %(backend_list)s') % + {'backend': backend, + 'backend_list': ','.join(VALID_DOGPILE_BACKENDS.keys())}) else: self.driver = VALID_DOGPILE_BACKENDS[backend](arguments) + def __getattr__(self, name): + """Forward calls to the underlying driver.""" + f = getattr(self.driver, name) + setattr(self, name, f) + return f + def _get_set_arguments_driver_attr(self, exclude_expiry=False): # NOTE(morganfainberg): Shallow copy the .set_arguments dict to diff --git a/keystone-moon/keystone/common/kvs/core.py b/keystone-moon/keystone/common/kvs/core.py index cbbb7462..6ce7b318 100644 --- a/keystone-moon/keystone/common/kvs/core.py +++ b/keystone-moon/keystone/common/kvs/core.py @@ -25,7 +25,6 @@ from dogpile.core import nameregistry from oslo_config import cfg from oslo_log import log from oslo_utils import importutils -import six from keystone import exception from keystone.i18n import _ @@ -147,24 +146,24 @@ class KeyValueStore(object): self._region.name) def _set_keymangler_on_backend(self, key_mangler): - try: - self._region.backend.key_mangler = key_mangler - except Exception as e: - # NOTE(morganfainberg): The setting of the key_mangler on the - # backend is used to allow the backend to - # calculate a hashed key value as needed. Not all backends - # require the ability to calculate hashed keys. If the - # backend does not support/require this feature log a - # debug line and move on otherwise raise the proper exception. - # Support of the feature is implied by the existence of the - # 'raw_no_expiry_keys' attribute. - if not hasattr(self._region.backend, 'raw_no_expiry_keys'): - LOG.debug(('Non-expiring keys not supported/required by ' - '%(region)s backend; unable to set ' - 'key_mangler for backend: %(err)s'), - {'region': self._region.name, 'err': e}) - else: - raise + try: + self._region.backend.key_mangler = key_mangler + except Exception as e: + # NOTE(morganfainberg): The setting of the key_mangler on the + # backend is used to allow the backend to + # calculate a hashed key value as needed. Not all backends + # require the ability to calculate hashed keys. If the + # backend does not support/require this feature log a + # debug line and move on otherwise raise the proper exception. + # Support of the feature is implied by the existence of the + # 'raw_no_expiry_keys' attribute. + if not hasattr(self._region.backend, 'raw_no_expiry_keys'): + LOG.debug(('Non-expiring keys not supported/required by ' + '%(region)s backend; unable to set ' + 'key_mangler for backend: %(err)s'), + {'region': self._region.name, 'err': e}) + else: + raise def _set_key_mangler(self, key_mangler): # Set the key_mangler that is appropriate for the given region being @@ -232,7 +231,7 @@ class KeyValueStore(object): if config_args['lock_timeout'] > 0: config_args['lock_timeout'] += LOCK_WINDOW - for argument, value in six.iteritems(config_args): + for argument, value in config_args.items(): arg_key = '.'.join([prefix, 'arguments', argument]) conf_dict[arg_key] = value diff --git a/keystone-moon/keystone/common/kvs/legacy.py b/keystone-moon/keystone/common/kvs/legacy.py index ba036016..7e27d97f 100644 --- a/keystone-moon/keystone/common/kvs/legacy.py +++ b/keystone-moon/keystone/common/kvs/legacy.py @@ -12,8 +12,9 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_log import versionutils + from keystone import exception -from keystone.openstack.common import versionutils class DictKvs(dict): diff --git a/keystone-moon/keystone/common/ldap/core.py b/keystone-moon/keystone/common/ldap/core.py index 144c0cfd..0bb3830c 100644 --- a/keystone-moon/keystone/common/ldap/core.py +++ b/keystone-moon/keystone/common/ldap/core.py @@ -24,11 +24,13 @@ import ldap.filter import ldappool from oslo_log import log import six +from six.moves import map, zip from keystone import exception from keystone.i18n import _ from keystone.i18n import _LW + LOG = log.getLogger(__name__) LDAP_VALUES = {'TRUE': True, 'FALSE': False} @@ -159,7 +161,7 @@ def convert_ldap_result(ldap_result): at_least_one_referral = True continue - for kind, values in six.iteritems(attrs): + for kind, values in attrs.items(): try: val2py = enabled2py if kind == 'enabled' else ldap2py ldap_attrs[kind] = [val2py(x) for x in values] @@ -327,7 +329,7 @@ def dn_startswith(descendant_dn, dn): @six.add_metaclass(abc.ABCMeta) class LDAPHandler(object): - '''Abstract class which defines methods for a LDAP API provider. + """Abstract class which defines methods for a LDAP API provider. Native Keystone values cannot be passed directly into and from the python-ldap API. Type conversion must occur at the LDAP API @@ -415,7 +417,8 @@ class LDAPHandler(object): method to any derivations of the abstract class the code will fail to load and run making it impossible to forget updating all the derived classes. - ''' + + """ @abc.abstractmethod def __init__(self, conn=None): self.conn = conn @@ -481,13 +484,13 @@ class LDAPHandler(object): class PythonLDAPHandler(LDAPHandler): - '''Implementation of the LDAPHandler interface which calls the - python-ldap API. + """LDAPHandler implementation which calls the python-ldap API. - Note, the python-ldap API requires all string values to be UTF-8 - encoded. The KeystoneLDAPHandler enforces this prior to invoking - the methods in this class. - ''' + Note, the python-ldap API requires all string values to be UTF-8 encoded. + The KeystoneLDAPHandler enforces this prior to invoking the methods in this + class. + + """ def __init__(self, conn=None): super(PythonLDAPHandler, self).__init__(conn=conn) @@ -569,10 +572,7 @@ class PythonLDAPHandler(LDAPHandler): def _common_ldap_initialization(url, use_tls=False, tls_cacertfile=None, tls_cacertdir=None, tls_req_cert=None, debug_level=None): - '''Method for common ldap initialization between PythonLDAPHandler and - PooledLDAPHandler. - ''' - + """LDAP initialization for PythonLDAPHandler and PooledLDAPHandler.""" LOG.debug("LDAP init: url=%s", url) LOG.debug('LDAP init: use_tls=%s tls_cacertfile=%s tls_cacertdir=%s ' 'tls_req_cert=%s tls_avail=%s', @@ -616,7 +616,7 @@ def _common_ldap_initialization(url, use_tls=False, tls_cacertfile=None, "or is not a directory") % tls_cacertdir) ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, tls_cacertdir) - if tls_req_cert in LDAP_TLS_CERTS.values(): + if tls_req_cert in list(LDAP_TLS_CERTS.values()): ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, tls_req_cert) else: LOG.debug("LDAP TLS: invalid TLS_REQUIRE_CERT Option=%s", @@ -624,15 +624,16 @@ def _common_ldap_initialization(url, use_tls=False, tls_cacertfile=None, class MsgId(list): - '''Wrapper class to hold connection and msgid.''' + """Wrapper class to hold connection and msgid.""" pass def use_conn_pool(func): - '''Use this only for connection pool specific ldap API. + """Use this only for connection pool specific ldap API. This adds connection object to decorated API as next argument after self. - ''' + + """ def wrapper(self, *args, **kwargs): # assert isinstance(self, PooledLDAPHandler) with self._get_pool_connection() as conn: @@ -642,8 +643,7 @@ def use_conn_pool(func): class PooledLDAPHandler(LDAPHandler): - '''Implementation of the LDAPHandler interface which uses pooled - connection manager. + """LDAPHandler implementation which uses pooled connection manager. Pool specific configuration is defined in [ldap] section. All other LDAP configuration is still used from [ldap] section @@ -663,8 +663,8 @@ class PooledLDAPHandler(LDAPHandler): Note, the python-ldap API requires all string values to be UTF-8 encoded. The KeystoneLDAPHandler enforces this prior to invoking the methods in this class. - ''' + """ # Added here to allow override for testing Connector = ldappool.StateConnector auth_pool_prefix = 'auth_pool_' @@ -737,7 +737,7 @@ class PooledLDAPHandler(LDAPHandler): # if connection has a lifetime, then it already has options specified if conn.get_lifetime() > 30: return - for option, invalue in six.iteritems(self.conn_options): + for option, invalue in self.conn_options.items(): conn.set_option(option, invalue) def _get_pool_connection(self): @@ -745,9 +745,8 @@ class PooledLDAPHandler(LDAPHandler): def simple_bind_s(self, who='', cred='', serverctrls=None, clientctrls=None): - '''Not using use_conn_pool decorator here as this API takes cred as - input. - ''' + # Not using use_conn_pool decorator here as this API takes cred as + # input. self.who = who self.cred = cred with self._get_pool_connection() as conn: @@ -773,16 +772,17 @@ class PooledLDAPHandler(LDAPHandler): filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0): - '''This API is asynchoronus API which returns MsgId instance to be used - in result3 call. + """Asynchronous API to return a ``MsgId`` instance. + + The ``MsgId`` instance can be safely used in a call to ``result3()``. - To work with result3 API in predicatable manner, same LDAP connection - is needed which provided msgid. So wrapping used connection and msgid - in MsgId class. The connection associated with search_ext is released - once last hard reference to MsgId object is freed. This will happen - when the method is done with returned MsgId usage. - ''' + To work with ``result3()`` API in predictable manner, the same LDAP + connection is needed which originally provided the ``msgid``. So, this + method wraps the existing connection and ``msgid`` in a new ``MsgId`` + instance. The connection associated with ``search_ext`` is released + once last hard reference to the ``MsgId`` instance is freed. + """ conn_ctxt = self._get_pool_connection() conn = conn_ctxt.__enter__() try: @@ -800,11 +800,12 @@ class PooledLDAPHandler(LDAPHandler): def result3(self, msgid, all=1, timeout=None, resp_ctrl_classes=None): - '''This method is used to wait for and return the result of an - operation previously initiated by one of the LDAP asynchronous - operation routines (eg search_ext()) It returned an invocation - identifier (a message id) upon successful initiation of their - operation. + """This method is used to wait for and return result. + + This method returns the result of an operation previously initiated by + one of the LDAP asynchronous operation routines (eg search_ext()). It + returned an invocation identifier (a message id) upon successful + initiation of their operation. Input msgid is expected to be instance of class MsgId which has LDAP session/connection used to execute search_ext and message idenfier. @@ -812,7 +813,8 @@ class PooledLDAPHandler(LDAPHandler): The connection associated with search_ext is released once last hard reference to MsgId object is freed. This will happen when function which requested msgId and used it in result3 exits. - ''' + + """ conn, msg_id = msgid return conn.result3(msg_id, all, timeout) @@ -831,7 +833,7 @@ class PooledLDAPHandler(LDAPHandler): class KeystoneLDAPHandler(LDAPHandler): - '''Convert data types and perform logging. + """Convert data types and perform logging. This LDAP inteface wraps the python-ldap based interfaces. The python-ldap interfaces require string values encoded in UTF-8. The @@ -854,7 +856,8 @@ class KeystoneLDAPHandler(LDAPHandler): Data returned from the LDAP call is converted back from UTF-8 encoded strings into the Python data type used internally in OpenStack. - ''' + + """ def __init__(self, conn=None): super(KeystoneLDAPHandler, self).__init__(conn=conn) @@ -938,7 +941,7 @@ class KeystoneLDAPHandler(LDAPHandler): if attrlist is None: attrlist_utf8 = None else: - attrlist_utf8 = map(utf8_encode, attrlist) + attrlist_utf8 = list(map(utf8_encode, attrlist)) ldap_result = self.conn.search_s(base_utf8, scope, filterstr_utf8, attrlist_utf8, attrsonly) @@ -989,7 +992,7 @@ class KeystoneLDAPHandler(LDAPHandler): attrlist_utf8 = None else: attrlist = [attr for attr in attrlist if attr is not None] - attrlist_utf8 = map(utf8_encode, attrlist) + attrlist_utf8 = list(map(utf8_encode, attrlist)) msgid = self.conn.search_ext(base_utf8, scope, filterstr_utf8, @@ -1083,7 +1086,7 @@ def register_handler(prefix, handler): def _get_connection(conn_url, use_pool=False, use_auth_pool=False): - for prefix, handler in six.iteritems(_HANDLERS): + for prefix, handler in _HANDLERS.items(): if conn_url.startswith(prefix): return handler() @@ -1109,7 +1112,6 @@ def filter_entity(entity_ref): class BaseLdap(object): - DEFAULT_SUFFIX = "dc=example,dc=com" DEFAULT_OU = None DEFAULT_STRUCTURAL_CLASSES = None DEFAULT_ID_ATTR = 'cn' @@ -1156,8 +1158,6 @@ class BaseLdap(object): if self.options_name is not None: self.suffix = conf.ldap.suffix - if self.suffix is None: - self.suffix = self.DEFAULT_SUFFIX dn = '%s_tree_dn' % self.options_name self.tree_dn = (getattr(conf.ldap, dn) or '%s,%s' % (self.DEFAULT_OU, self.suffix)) @@ -1169,7 +1169,7 @@ class BaseLdap(object): self.object_class = (getattr(conf.ldap, objclass) or self.DEFAULT_OBJECTCLASS) - for k, v in six.iteritems(self.attribute_options_names): + for k, v in self.attribute_options_names.items(): v = '%s_%s_attribute' % (self.options_name, v) self.attribute_mapping[k] = getattr(conf.ldap, v) @@ -1318,7 +1318,7 @@ class BaseLdap(object): # in a case-insensitive way. We use the case specified in the # mapping for the model to ensure we have a predictable way of # retrieving values later. - lower_res = {k.lower(): v for k, v in six.iteritems(res[1])} + lower_res = {k.lower(): v for k, v in res[1].items()} id_attrs = lower_res.get(self.id_attr.lower()) if not id_attrs: @@ -1404,7 +1404,7 @@ class BaseLdap(object): self.affirm_unique(values) object_classes = self.structural_classes + [self.object_class] attrs = [('objectClass', object_classes)] - for k, v in six.iteritems(values): + for k, v in values.items(): if k in self.attribute_ignore: continue if k == 'id': @@ -1416,7 +1416,7 @@ class BaseLdap(object): if attr_type is not None: attrs.append((attr_type, [v])) extra_attrs = [attr for attr, name - in six.iteritems(self.extra_attr_mapping) + in self.extra_attr_mapping.items() if name == k] for attr in extra_attrs: attrs.append((attr, [v])) @@ -1439,8 +1439,8 @@ class BaseLdap(object): with self.get_connection() as conn: try: attrs = list(set(([self.id_attr] + - self.attribute_mapping.values() + - self.extra_attr_mapping.keys()))) + list(self.attribute_mapping.values()) + + list(self.extra_attr_mapping.keys())))) res = conn.search_s(self.tree_dn, self.LDAP_SCOPE, query, @@ -1453,14 +1453,15 @@ class BaseLdap(object): return None def _ldap_get_all(self, ldap_filter=None): - query = u'(&%s(objectClass=%s))' % (ldap_filter or - self.ldap_filter or - '', self.object_class) + query = u'(&%s(objectClass=%s)(%s=*))' % ( + ldap_filter or self.ldap_filter or '', + self.object_class, + self.id_attr) with self.get_connection() as conn: try: attrs = list(set(([self.id_attr] + - self.attribute_mapping.values() + - self.extra_attr_mapping.keys()))) + list(self.attribute_mapping.values()) + + list(self.extra_attr_mapping.keys())))) return conn.search_s(self.tree_dn, self.LDAP_SCOPE, query, @@ -1479,7 +1480,7 @@ class BaseLdap(object): query = (u'(&%s%s)' % (query, ''.join([calc_filter(k, v) for k, v in - six.iteritems(query_params)]))) + query_params.items()]))) with self.get_connection() as conn: return conn.search_s(search_base, scope, query, attrlist) @@ -1509,7 +1510,7 @@ class BaseLdap(object): old_obj = self.get(object_id) modlist = [] - for k, v in six.iteritems(values): + for k, v in values.items(): if k == 'id': # id can't be modified. continue @@ -1648,7 +1649,7 @@ class BaseLdap(object): (query, ''.join(['(%s=%s)' % (k, ldap.filter.escape_filter_chars(v)) for k, v in - six.iteritems(query_params)]))) + query_params.items()]))) not_deleted_nodes = [] with self.get_connection() as conn: try: @@ -1738,6 +1739,11 @@ class BaseLdap(object): return query_term + if query is None: + # make sure query is a string so the ldap filter is properly + # constructed from filter_list later + query = '' + if hints is None: return query @@ -1799,25 +1805,24 @@ class EnabledEmuMixIn(BaseLdap): utf8_decode(naming_rdn[1])) self.enabled_emulation_naming_attr = naming_attr - def _get_enabled(self, object_id): + def _get_enabled(self, object_id, conn): dn = self._id_to_dn(object_id) query = '(member=%s)' % dn - with self.get_connection() as conn: - try: - enabled_value = conn.search_s(self.enabled_emulation_dn, - ldap.SCOPE_BASE, - query, ['cn']) - except ldap.NO_SUCH_OBJECT: - return False - else: - return bool(enabled_value) + try: + enabled_value = conn.search_s(self.enabled_emulation_dn, + ldap.SCOPE_BASE, + query, attrlist=DN_ONLY) + except ldap.NO_SUCH_OBJECT: + return False + else: + return bool(enabled_value) def _add_enabled(self, object_id): - if not self._get_enabled(object_id): - modlist = [(ldap.MOD_ADD, - 'member', - [self._id_to_dn(object_id)])] - with self.get_connection() as conn: + with self.get_connection() as conn: + if not self._get_enabled(object_id, conn): + modlist = [(ldap.MOD_ADD, + 'member', + [self._id_to_dn(object_id)])] try: conn.modify_s(self.enabled_emulation_dn, modlist) except ldap.NO_SUCH_OBJECT: @@ -1851,10 +1856,12 @@ class EnabledEmuMixIn(BaseLdap): return super(EnabledEmuMixIn, self).create(values) def get(self, object_id, ldap_filter=None): - ref = super(EnabledEmuMixIn, self).get(object_id, ldap_filter) - if 'enabled' not in self.attribute_ignore and self.enabled_emulation: - ref['enabled'] = self._get_enabled(object_id) - return ref + with self.get_connection() as conn: + ref = super(EnabledEmuMixIn, self).get(object_id, ldap_filter) + if ('enabled' not in self.attribute_ignore and + self.enabled_emulation): + ref['enabled'] = self._get_enabled(object_id, conn) + return ref def get_all(self, ldap_filter=None): if 'enabled' not in self.attribute_ignore and self.enabled_emulation: @@ -1862,8 +1869,10 @@ class EnabledEmuMixIn(BaseLdap): tenant_list = [self._ldap_res_to_model(x) for x in self._ldap_get_all(ldap_filter) if x[0] != self.enabled_emulation_dn] - for tenant_ref in tenant_list: - tenant_ref['enabled'] = self._get_enabled(tenant_ref['id']) + with self.get_connection() as conn: + for tenant_ref in tenant_list: + tenant_ref['enabled'] = self._get_enabled( + tenant_ref['id'], conn) return tenant_list else: return super(EnabledEmuMixIn, self).get_all(ldap_filter) diff --git a/keystone-moon/keystone/common/manager.py b/keystone-moon/keystone/common/manager.py index 28bf2efb..7150fbf3 100644 --- a/keystone-moon/keystone/common/manager.py +++ b/keystone-moon/keystone/common/manager.py @@ -14,7 +14,13 @@ import functools +from oslo_log import log +from oslo_log import versionutils from oslo_utils import importutils +import stevedore + + +LOG = log.getLogger(__name__) def response_truncated(f): @@ -53,6 +59,28 @@ def response_truncated(f): return wrapper +def load_driver(namespace, driver_name, *args): + try: + driver_manager = stevedore.DriverManager(namespace, + driver_name, + invoke_on_load=True, + invoke_args=args) + return driver_manager.driver + except RuntimeError as e: + LOG.debug('Failed to load %r using stevedore: %s', driver_name, e) + # Ignore failure and continue on. + + @versionutils.deprecated(as_of=versionutils.deprecated.LIBERTY, + in_favor_of='entrypoints', + what='direct import of driver') + def _load_using_import(driver_name, *args): + return importutils.import_object(driver_name, *args) + + # For backwards-compatibility, an unregistered class reference can + # still be used. + return _load_using_import(driver_name, *args) + + class Manager(object): """Base class for intermediary request layer. @@ -66,8 +94,10 @@ class Manager(object): """ + driver_namespace = None + def __init__(self, driver_name): - self.driver = importutils.import_object(driver_name) + self.driver = load_driver(self.driver_namespace, driver_name) def __getattr__(self, name): """Forward calls to the underlying driver.""" diff --git a/keystone-moon/keystone/common/models.py b/keystone-moon/keystone/common/models.py index 3b3aabe1..0bb37319 100644 --- a/keystone-moon/keystone/common/models.py +++ b/keystone-moon/keystone/common/models.py @@ -130,11 +130,12 @@ class Project(Model): Optional Keys: description enabled (bool, default True) + is_domain (bool, default False) """ required_keys = ('id', 'name', 'domain_id') - optional_keys = ('description', 'enabled') + optional_keys = ('description', 'enabled', 'is_domain') class Role(Model): diff --git a/keystone-moon/keystone/common/openssl.py b/keystone-moon/keystone/common/openssl.py index 4eb7d1d1..be56b9cc 100644 --- a/keystone-moon/keystone/common/openssl.py +++ b/keystone-moon/keystone/common/openssl.py @@ -20,7 +20,7 @@ from oslo_log import log from keystone.common import environment from keystone.common import utils -from keystone.i18n import _LI, _LE +from keystone.i18n import _LI, _LE, _LW LOG = log.getLogger(__name__) CONF = cfg.CONF @@ -70,8 +70,8 @@ class BaseCertificateConfigure(object): if "OpenSSL 0." in openssl_ver: self.ssl_dictionary['default_md'] = 'sha1' except OSError: - LOG.warn('Failed to invoke ``openssl version``, ' - 'assuming is v1.0 or newer') + LOG.warn(_LW('Failed to invoke ``openssl version``, ' + 'assuming is v1.0 or newer')) self.ssl_dictionary.update(kwargs) def exec_command(self, command): diff --git a/keystone-moon/keystone/common/sql/core.py b/keystone-moon/keystone/common/sql/core.py index bf168701..ebd61bb7 100644 --- a/keystone-moon/keystone/common/sql/core.py +++ b/keystone-moon/keystone/common/sql/core.py @@ -239,6 +239,39 @@ def truncated(f): return wrapper +class _WontMatch(Exception): + """Raised to indicate that the filter won't match. + + This is raised to short-circuit the computation of the filter as soon as + it's discovered that the filter requested isn't going to match anything. + + A filter isn't going to match anything if the value is too long for the + field, for example. + + """ + + @classmethod + def check(cls, value, col_attr): + """Check if the value can match given the column attributes. + + Raises this class if the value provided can't match any value in the + column in the table given the column's attributes. For example, if the + column is a string and the value is longer than the column then it + won't match any value in the column in the table. + + """ + col = col_attr.property.columns[0] + if isinstance(col.type, sql.types.Boolean): + # The column is a Boolean, we should have already validated input. + return + if not col.type.length: + # The column doesn't have a length so can't validate anymore. + return + if len(value) > col.type.length: + raise cls() + # Otherwise the value could match a value in the column. + + def _filter(model, query, hints): """Applies filtering to a query. @@ -251,16 +284,14 @@ def _filter(model, query, hints): :returns query: query, updated with any filters satisfied """ - def inexact_filter(model, query, filter_, satisfied_filters, hints): + def inexact_filter(model, query, filter_, satisfied_filters): """Applies an inexact filter to a query. :param model: the table model in question :param query: query to apply filters to - :param filter_: the dict that describes this filter - :param satisfied_filters: a cumulative list of satisfied filters, to - which filter_ will be added if it is - satisfied. - :param hints: contains the list of filters yet to be satisfied. + :param dict filter_: describes this filter + :param list satisfied_filters: filter_ will be added if it is + satisfied. :returns query: query updated to add any inexact filters we could satisfy @@ -278,10 +309,13 @@ def _filter(model, query, hints): return query if filter_['comparator'] == 'contains': + _WontMatch.check(filter_['value'], column_attr) query_term = column_attr.ilike('%%%s%%' % filter_['value']) elif filter_['comparator'] == 'startswith': + _WontMatch.check(filter_['value'], column_attr) query_term = column_attr.ilike('%s%%' % filter_['value']) elif filter_['comparator'] == 'endswith': + _WontMatch.check(filter_['value'], column_attr) query_term = column_attr.ilike('%%%s' % filter_['value']) else: # It's a filter we don't understand, so let the caller @@ -291,53 +325,50 @@ def _filter(model, query, hints): satisfied_filters.append(filter_) return query.filter(query_term) - def exact_filter( - model, filter_, satisfied_filters, cumulative_filter_dict, hints): + def exact_filter(model, filter_, cumulative_filter_dict): """Applies an exact filter to a query. :param model: the table model in question - :param filter_: the dict that describes this filter - :param satisfied_filters: a cumulative list of satisfied filters, to - which filter_ will be added if it is - satisfied. - :param cumulative_filter_dict: a dict that describes the set of - exact filters built up so far - :param hints: contains the list of filters yet to be satisfied. - - :returns: updated cumulative dict + :param dict filter_: describes this filter + :param dict cumulative_filter_dict: describes the set of exact filters + built up so far """ key = filter_['name'] - if isinstance(getattr(model, key).property.columns[0].type, - sql.types.Boolean): + + col = getattr(model, key) + if isinstance(col.property.columns[0].type, sql.types.Boolean): cumulative_filter_dict[key] = ( utils.attr_as_boolean(filter_['value'])) else: + _WontMatch.check(filter_['value'], col) cumulative_filter_dict[key] = filter_['value'] - satisfied_filters.append(filter_) - return cumulative_filter_dict - - filter_dict = {} - satisfied_filters = [] - for filter_ in hints.filters: - if filter_['name'] not in model.attributes: - continue - if filter_['comparator'] == 'equals': - filter_dict = exact_filter( - model, filter_, satisfied_filters, filter_dict, hints) - else: - query = inexact_filter( - model, query, filter_, satisfied_filters, hints) - # Apply any exact filters we built up - if filter_dict: - query = query.filter_by(**filter_dict) + try: + filter_dict = {} + satisfied_filters = [] + for filter_ in hints.filters: + if filter_['name'] not in model.attributes: + continue + if filter_['comparator'] == 'equals': + exact_filter(model, filter_, filter_dict) + satisfied_filters.append(filter_) + else: + query = inexact_filter(model, query, filter_, + satisfied_filters) + + # Apply any exact filters we built up + if filter_dict: + query = query.filter_by(**filter_dict) + + # Remove satisfied filters, then the caller will know remaining filters + for filter_ in satisfied_filters: + hints.filters.remove(filter_) - # Remove satisfied filters, then the caller will know remaining filters - for filter_ in satisfied_filters: - hints.filters.remove(filter_) - - return query + return query + except _WontMatch: + hints.cannot_match = True + return def _limit(query, hints): @@ -378,6 +409,10 @@ def filter_limit_query(model, query, hints): # First try and satisfy any filters query = _filter(model, query, hints) + if hints.cannot_match: + # Nothing's going to match, so don't bother with the query. + return [] + # NOTE(henry-nash): Any unsatisfied filters will have been left in # the hints list for the controller to handle. We can only try and # limit here if all the filters are already satisfied since, if not, diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py index 535a0944..c4b41580 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py @@ -27,7 +27,8 @@ def upgrade(migrate_engine): # names, depending on version of MySQL used. We shoud make this naming # consistent, by reverting index name to a consistent condition. if any(i for i in endpoint.indexes if - i.columns.keys() == ['service_id'] and i.name != 'service_id'): + list(i.columns.keys()) == ['service_id'] + and i.name != 'service_id'): # NOTE(i159): by this action will be made re-creation of an index # with the new name. This can be considered as renaming under the # MySQL rules. @@ -37,13 +38,6 @@ def upgrade(migrate_engine): meta, autoload=True) if any(i for i in user_group_membership.indexes if - i.columns.keys() == ['group_id'] and i.name != 'group_id'): + list(i.columns.keys()) == ['group_id'] + and i.name != 'group_id'): sa.Index('group_id', user_group_membership.c.group_id).create() - - -def downgrade(migrate_engine): - # NOTE(i159): index exists only in MySQL schemas, and got an inconsistent - # name only when MySQL 5.5 renamed it after re-creation - # (during migrations). So we just fixed inconsistency, there is no - # necessity to revert it. - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py index 074fbb63..59720f6e 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py @@ -39,11 +39,3 @@ def upgrade(migrate_engine): mysql_engine='InnoDB', mysql_charset='utf8') mapping_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - assignment = sql.Table(MAPPING_TABLE, meta, autoload=True) - assignment.drop(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py index 9f1fd9f0..86302a8f 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py @@ -14,6 +14,7 @@ import sqlalchemy as sql + _REGION_TABLE_NAME = 'region' @@ -24,11 +25,3 @@ def upgrade(migrate_engine): region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) url_column = sql.Column('url', sql.String(255), nullable=True) region_table.create_column(url_column) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) - region_table.drop_column('url') diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py index 6dc0004f..c2be48f4 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py @@ -12,7 +12,6 @@ # License for the specific language governing permissions and limitations # under the License. - """Migrated the endpoint 'region' column to 'region_id. In addition to the rename, the new column is made a foreign key to the @@ -36,25 +35,9 @@ b. For each endpoint ii. Assign the id to the region_id column c. Remove the column region - -To Downgrade: - -Endpoint Table - -a. Add back in the region column -b. For each endpoint - i. Copy the region_id column to the region column -c. Remove the column region_id - -Region Table - -Decrease the size of the id column in the region table, making sure that -we don't get classing primary keys. - """ import migrate -import six import sqlalchemy as sql from sqlalchemy.orm import sessionmaker @@ -90,39 +73,6 @@ def _migrate_to_region_id(migrate_engine, region_table, endpoint_table): name='fk_endpoint_region_id').create() -def _migrate_to_region(migrate_engine, region_table, endpoint_table): - endpoints = list(endpoint_table.select().execute()) - - for endpoint in endpoints: - new_values = {'region': endpoint.region_id} - f = endpoint_table.c.id == endpoint.id - update = endpoint_table.update().where(f).values(new_values) - migrate_engine.execute(update) - - if 'sqlite' != migrate_engine.name: - migrate.ForeignKeyConstraint( - columns=[endpoint_table.c.region_id], - refcolumns=[region_table.c.id], - name='fk_endpoint_region_id').drop() - endpoint_table.c.region_id.drop() - - -def _prepare_regions_for_id_truncation(migrate_engine, region_table): - """Ensure there are no IDs that are bigger than 64 chars. - - The size of the id and parent_id fields where increased from 64 to 255 - during the upgrade. On downgrade we have to make sure that the ids can - fit in the new column size. For rows with ids greater than this, we have - no choice but to dump them. - - """ - for region in list(region_table.select().execute()): - if (len(six.text_type(region.id)) > 64 or - len(six.text_type(region.parent_region_id)) > 64): - delete = region_table.delete(region_table.c.id == region.id) - migrate_engine.execute(delete) - - def upgrade(migrate_engine): meta = sql.MetaData() meta.bind = migrate_engine @@ -138,19 +88,3 @@ def upgrade(migrate_engine): _migrate_to_region_id(migrate_engine, region_table, endpoint_table) endpoint_table.c.region.drop() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table('region', meta, autoload=True) - endpoint_table = sql.Table('endpoint', meta, autoload=True) - region_column = sql.Column('region', sql.String(length=255)) - region_column.create(endpoint_table) - - _migrate_to_region(migrate_engine, region_table, endpoint_table) - _prepare_regions_for_id_truncation(migrate_engine, region_table) - - region_table.c.id.alter(type=sql.String(length=64)) - region_table.c.parent_region_id.alter(type=sql.String(length=64)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py index 33b13b7d..caf4d66f 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py @@ -14,6 +14,7 @@ import sqlalchemy as sql + ASSIGNMENT_TABLE = 'assignment' @@ -24,12 +25,3 @@ def upgrade(migrate_engine): assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True) idx = sql.Index('ix_actor_id', assignment.c.actor_id) idx.create(migrate_engine) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True) - idx = sql.Index('ix_actor_id', assignment.c.actor_id) - idx.drop(migrate_engine) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py index 1cfddd3f..a7f327ea 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py @@ -23,13 +23,3 @@ def upgrade(migrate_engine): sql.Index('ix_token_user_id', token.c.user_id).create() sql.Index('ix_token_trust_id', token.c.trust_id).create() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - token = sql.Table('token', meta, autoload=True) - - sql.Index('ix_token_user_id', token.c.user_id).drop() - sql.Index('ix_token_trust_id', token.c.trust_id).drop() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py index bb8ef9f6..ca9b3ce2 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py @@ -14,6 +14,7 @@ import sqlalchemy as sql from keystone.common.sql import migration_helpers + _PROJECT_TABLE_NAME = 'project' _PARENT_ID_COLUMN_NAME = 'parent_id' @@ -38,17 +39,3 @@ def upgrade(migrate_engine): if migrate_engine.name == 'sqlite': return migration_helpers.add_constraints(list_constraints(project_table)) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) - - # SQLite does not support constraints, and querying the constraints - # raises an exception - if migrate_engine.name != 'sqlite': - migration_helpers.remove_constraints(list_constraints(project_table)) - - project_table.drop_column(_PARENT_ID_COLUMN_NAME) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py index 5a33486c..f7a69bb6 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py @@ -33,9 +33,3 @@ def upgrade(migrate_engine): if migrate_engine.name == 'sqlite': return migration_helpers.remove_constraints(list_constraints(migrate_engine)) - - -def downgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - return - migration_helpers.add_constraints(list_constraints(migrate_engine)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py index 109a8412..e45133ab 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py @@ -12,6 +12,7 @@ import sqlalchemy as sql + _REGION_TABLE_NAME = 'region' @@ -21,12 +22,3 @@ def upgrade(migrate_engine): region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) region_table.drop_column('url') - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) - url_column = sql.Column('url', sql.String(255), nullable=True) - region_table.create_column(url_column) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py index bca00902..637f2151 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py @@ -37,9 +37,3 @@ def upgrade(migrate_engine): if migrate_engine.name == 'sqlite': return migration_helpers.remove_constraints(list_constraints(migrate_engine)) - - -def downgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - return - migration_helpers.add_constraints(list_constraints(migrate_engine)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py index fd8717d2..63a86c11 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py @@ -14,6 +14,7 @@ import sqlalchemy as sql from keystone.common import sql as ks_sql + WHITELIST_TABLE = 'whitelisted_config' SENSITIVE_TABLE = 'sensitive_config' @@ -43,13 +44,3 @@ def upgrade(migrate_engine): mysql_engine='InnoDB', mysql_charset='utf8') sensitive_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - table = sql.Table(WHITELIST_TABLE, meta, autoload=True) - table.drop(migrate_engine, checkfirst=True) - table = sql.Table(SENSITIVE_TABLE, meta, autoload=True) - table.drop(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py index 3feadc53..fe0cee88 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py @@ -22,7 +22,11 @@ def upgrade(migrate_engine): services = list(service_table.select().execute()) for service in services: - extra_dict = jsonutils.loads(service.extra) + if service.extra is not None: + extra_dict = jsonutils.loads(service.extra) + else: + extra_dict = {} + # Skip records where service is not null if extra_dict.get('name') is not None: continue @@ -34,10 +38,3 @@ def upgrade(migrate_engine): f = service_table.c.id == service.id update = service_table.update().where(f).values(new_values) migrate_engine.execute(update) - - -def downgrade(migration_engine): - # The upgrade fixes the data inconsistency for the service name, - # it defaults the value to empty string. There is no necessity - # to revert it. - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py new file mode 100644 index 00000000..b9df1a55 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy + + +def upgrade(migrate_engine): + # NOTE(viktors): Migration 062 removed FK from `assignment` table, but + # MySQL silently creates indexes on FK constraints, so we should remove + # this index manually. + if migrate_engine.name == 'mysql': + meta = sqlalchemy.MetaData(bind=migrate_engine) + table = sqlalchemy.Table('assignment', meta, autoload=True) + for index in table.indexes: + if [c.name for c in index.columns] == ['role_id']: + index.drop(migrate_engine) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/068_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/068_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/068_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/069_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/069_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/069_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/070_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/070_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/070_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/071_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/071_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/071_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/072_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/072_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/072_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py new file mode 100644 index 00000000..ffa210c4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py @@ -0,0 +1,114 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy as sql +from sqlalchemy.orm import sessionmaker + +from keystone.assignment.backends import sql as assignment_sql + + +def upgrade(migrate_engine): + """Inserts inherited column to assignment table PK contraints. + + For non-SQLite databases, it changes the constraint in the existing table. + + For SQLite, since changing constraints is not supported, it recreates the + assignment table with the new PK constraint and migrates the existing data. + + """ + + ASSIGNMENT_TABLE_NAME = 'assignment' + + metadata = sql.MetaData() + metadata.bind = migrate_engine + + # Retrieve the existing assignment table + assignment_table = sql.Table(ASSIGNMENT_TABLE_NAME, metadata, + autoload=True) + + if migrate_engine.name == 'sqlite': + ACTOR_ID_INDEX_NAME = 'ix_actor_id' + TMP_ASSIGNMENT_TABLE_NAME = 'tmp_assignment' + + # Define the new assignment table with a temporary name + new_assignment_table = sql.Table( + TMP_ASSIGNMENT_TABLE_NAME, metadata, + sql.Column('type', sql.Enum( + assignment_sql.AssignmentType.USER_PROJECT, + assignment_sql.AssignmentType.GROUP_PROJECT, + assignment_sql.AssignmentType.USER_DOMAIN, + assignment_sql.AssignmentType.GROUP_DOMAIN, + name='type'), + nullable=False), + sql.Column('actor_id', sql.String(64), nullable=False), + sql.Column('target_id', sql.String(64), nullable=False), + sql.Column('role_id', sql.String(64), sql.ForeignKey('role.id'), + nullable=False), + sql.Column('inherited', sql.Boolean, default=False, + nullable=False), + sql.PrimaryKeyConstraint('type', 'actor_id', 'target_id', + 'role_id', 'inherited'), + mysql_engine='InnoDB', + mysql_charset='utf8') + + # Create the new assignment table + new_assignment_table.create(migrate_engine, checkfirst=True) + + # Change the index from the existing assignment table to the new one + sql.Index(ACTOR_ID_INDEX_NAME, assignment_table.c.actor_id).drop() + sql.Index(ACTOR_ID_INDEX_NAME, + new_assignment_table.c.actor_id).create() + + # Instantiate session + maker = sessionmaker(bind=migrate_engine) + session = maker() + + # Migrate existing data + insert = new_assignment_table.insert().from_select( + assignment_table.c, select=session.query(assignment_table)) + session.execute(insert) + session.commit() + + # Drop the existing assignment table, in favor of the new one + assignment_table.deregister() + assignment_table.drop() + + # Finally, rename the new table to the original assignment table name + new_assignment_table.rename(ASSIGNMENT_TABLE_NAME) + elif migrate_engine.name == 'ibm_db_sa': + # Recreate the existing constraint, marking the inherited column as PK + # for DB2. + + # This is a workaround to the general case in the else statement below. + # Due to a bug in the DB2 sqlalchemy dialect, Column.alter() actually + # creates a primary key over only the "inherited" column. This is wrong + # because the primary key for the table actually covers other columns + # too, not just the "inherited" column. Since the primary key already + # exists for the table after the Column.alter() call, it causes the + # next line to fail with an error that the primary key already exists. + + # The workaround here skips doing the Column.alter(). This causes a + # warning message since the metadata is out of sync. We can remove this + # workaround once the DB2 sqlalchemy dialect is fixed. + # DB2 Issue: https://code.google.com/p/ibm-db/issues/detail?id=173 + + migrate.PrimaryKeyConstraint(table=assignment_table).drop() + migrate.PrimaryKeyConstraint( + assignment_table.c.type, assignment_table.c.actor_id, + assignment_table.c.target_id, assignment_table.c.role_id, + assignment_table.c.inherited).create() + else: + # Recreate the existing constraint, marking the inherited column as PK + migrate.PrimaryKeyConstraint(table=assignment_table).drop() + assignment_table.c.inherited.alter(primary_key=True) + migrate.PrimaryKeyConstraint(table=assignment_table).create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/074_add_is_domain_project.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/074_add_is_domain_project.py new file mode 100644 index 00000000..dcb89b07 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/074_add_is_domain_project.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + + +_PROJECT_TABLE_NAME = 'project' +_IS_DOMAIN_COLUMN_NAME = 'is_domain' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) + is_domain = sql.Column(_IS_DOMAIN_COLUMN_NAME, sql.Boolean, nullable=False, + server_default='0', default=False) + project_table.create_column(is_domain) diff --git a/keystone-moon/keystone/common/sql/migration_helpers.py b/keystone-moon/keystone/common/sql/migration_helpers.py index 86932995..aaa59f70 100644 --- a/keystone-moon/keystone/common/sql/migration_helpers.py +++ b/keystone-moon/keystone/common/sql/migration_helpers.py @@ -143,37 +143,21 @@ def _sync_common_repo(version): abs_path = find_migrate_repo() init_version = migrate_repo.DB_INIT_VERSION engine = sql.get_engine() + _assert_not_schema_downgrade(version=version) migration.db_sync(engine, abs_path, version=version, - init_version=init_version) + init_version=init_version, sanity_check=False) -def _fix_federation_tables(engine): - """Fix the identity_provider, federation_protocol and mapping tables - to be InnoDB and Charset UTF8. - - This function is to work around bug #1426334. This has occurred because - the original migration did not specify InnoDB and charset utf8. Due - to the sanity_check, a deployer can get wedged here and require manual - database changes to fix. - """ - # NOTE(marco-fargetta) This is a workaround to "fix" that tables only - # if we're under MySQL - if engine.name == 'mysql': - # * Disable any check for the foreign keys because they prevent the - # alter table to execute - engine.execute("SET foreign_key_checks = 0") - # * Make the tables using InnoDB engine - engine.execute("ALTER TABLE identity_provider Engine=InnoDB") - engine.execute("ALTER TABLE federation_protocol Engine=InnoDB") - engine.execute("ALTER TABLE mapping Engine=InnoDB") - # * Make the tables using utf8 encoding - engine.execute("ALTER TABLE identity_provider " - "CONVERT TO CHARACTER SET utf8") - engine.execute("ALTER TABLE federation_protocol " - "CONVERT TO CHARACTER SET utf8") - engine.execute("ALTER TABLE mapping CONVERT TO CHARACTER SET utf8") - # * Revert the foreign keys check back - engine.execute("SET foreign_key_checks = 1") +def _assert_not_schema_downgrade(extension=None, version=None): + if version is not None: + try: + current_ver = int(six.text_type(get_db_version(extension))) + if int(version) < current_ver: + raise migration.exception.DbMigrationError() + except exceptions.DatabaseNotControlledError: + # NOTE(morganfainberg): The database is not controlled, this action + # cannot be a downgrade. + pass def _sync_extension_repo(extension, version): @@ -198,27 +182,11 @@ def _sync_extension_repo(extension, version): except exception.MigrationNotProvided as e: print(e) sys.exit(1) - try: - migration.db_sync(engine, abs_path, version=version, - init_version=init_version) - except ValueError: - # NOTE(marco-fargetta): ValueError is raised from the sanity check ( - # verifies that tables are utf8 under mysql). The federation_protocol, - # identity_provider and mapping tables were not initially built with - # InnoDB and utf8 as part of the table arguments when the migration - # was initially created. Bug #1426334 is a scenario where the deployer - # can get wedged, unable to upgrade or downgrade. - # This is a workaround to "fix" those tables if we're under MySQL and - # the version is before the 6 because before the tables were introduced - # before and patched when migration 5 was available - if engine.name == 'mysql' and \ - int(six.text_type(get_db_version(extension))) < 6: - _fix_federation_tables(engine) - # The migration is applied again after the fix - migration.db_sync(engine, abs_path, version=version, - init_version=init_version) - else: - raise + + _assert_not_schema_downgrade(extension=extension, version=version) + + migration.db_sync(engine, abs_path, version=version, + init_version=init_version, sanity_check=False) def sync_database_to_version(extension=None, version=None): diff --git a/keystone-moon/keystone/common/utils.py b/keystone-moon/keystone/common/utils.py index a4b03ffd..48336af7 100644 --- a/keystone-moon/keystone/common/utils.py +++ b/keystone-moon/keystone/common/utils.py @@ -27,10 +27,12 @@ from oslo_config import cfg from oslo_log import log from oslo_serialization import jsonutils from oslo_utils import strutils +from oslo_utils import timeutils import passlib.hash import six from six import moves +from keystone.common import authorization from keystone import exception from keystone.i18n import _, _LE, _LW @@ -51,7 +53,7 @@ def flatten_dict(d, parent_key=''): for k, v in d.items(): new_key = parent_key + '.' + k if parent_key else k if isinstance(v, collections.MutableMapping): - items.extend(flatten_dict(v, new_key).items()) + items.extend(list(flatten_dict(v, new_key).items())) else: items.append((new_key, v)) return dict(items) @@ -244,7 +246,7 @@ def setup_remote_pydev_debug(): def get_unix_user(user=None): - '''Get the uid and user name. + """Get the uid and user name. This is a convenience utility which accepts a variety of input which might represent a unix user. If successful it returns the uid @@ -257,7 +259,7 @@ def get_unix_user(user=None): lookup as a uid. int - An integer is interpretted as a uid. + An integer is interpreted as a uid. None None is interpreted to mean use the current process's @@ -270,7 +272,8 @@ def get_unix_user(user=None): lookup. :return: tuple of (uid, name) - ''' + + """ if isinstance(user, six.string_types): try: @@ -299,7 +302,7 @@ def get_unix_user(user=None): def get_unix_group(group=None): - '''Get the gid and group name. + """Get the gid and group name. This is a convenience utility which accepts a variety of input which might represent a unix group. If successful it returns the gid @@ -312,7 +315,7 @@ def get_unix_group(group=None): lookup as a gid. int - An integer is interpretted as a gid. + An integer is interpreted as a gid. None None is interpreted to mean use the current process's @@ -326,7 +329,8 @@ def get_unix_group(group=None): lookup. :return: tuple of (gid, name) - ''' + + """ if isinstance(group, six.string_types): try: @@ -357,7 +361,7 @@ def get_unix_group(group=None): def set_permissions(path, mode=None, user=None, group=None, log=None): - '''Set the ownership and permissions on the pathname. + """Set the ownership and permissions on the pathname. Each of the mode, user and group are optional, if None then that aspect is not modified. @@ -374,7 +378,8 @@ def set_permissions(path, mode=None, user=None, group=None, log=None): if None do not set. :param logger log: logging.logger object, used to emit log messages, if None no logging is performed. - ''' + + """ if user is None: user_uid, user_name = None, None @@ -420,7 +425,7 @@ def set_permissions(path, mode=None, user=None, group=None, log=None): def make_dirs(path, mode=None, user=None, group=None, log=None): - '''Assure directory exists, set ownership and permissions. + """Assure directory exists, set ownership and permissions. Assure the directory exists and optionally set its ownership and permissions. @@ -440,7 +445,8 @@ def make_dirs(path, mode=None, user=None, group=None, log=None): if None do not set. :param logger log: logging.logger object, used to emit log messages, if None no logging is performed. - ''' + + """ if log: if mode is None: @@ -469,3 +475,54 @@ class WhiteListedItemFilter(object): if name not in self._whitelist: raise KeyError return self._data[name] + + +_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' +_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' + + +def isotime(at=None, subsecond=False): + """Stringify time in ISO 8601 format.""" + + # Python provides a similar instance method for datetime.datetime objects + # called isoformat(). The format of the strings generated by isoformat() + # have a couple of problems: + # 1) The strings generated by isotime are used in tokens and other public + # APIs that we can't change without a deprecation period. The strings + # generated by isoformat are not the same format, so we can't just + # change to it. + # 2) The strings generated by isoformat do not include the microseconds if + # the value happens to be 0. This will likely show up as random failures + # as parsers may be written to always expect microseconds, and it will + # parse correctly most of the time. + + if not at: + at = timeutils.utcnow() + st = at.strftime(_ISO8601_TIME_FORMAT + if not subsecond + else _ISO8601_TIME_FORMAT_SUBSECOND) + tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' + st += ('Z' if tz == 'UTC' else tz) + return st + + +def strtime(): + at = timeutils.utcnow() + return at.strftime(timeutils.PERFECT_TIME_FORMAT) + + +def get_token_ref(context): + """Retrieves KeystoneToken object from the auth context and returns it. + + :param dict context: The request context. + :raises: exception.Unauthorized if auth context cannot be found. + :returns: The KeystoneToken object. + """ + try: + # Retrieve the auth context that was prepared by AuthContextMiddleware. + auth_context = (context['environment'] + [authorization.AUTH_CONTEXT_ENV]) + return auth_context['token'] + except KeyError: + LOG.warning(_LW("Couldn't find the auth context.")) + raise exception.Unauthorized() diff --git a/keystone-moon/keystone/common/validation/__init__.py b/keystone-moon/keystone/common/validation/__init__.py index f9c58eaf..1e5cc6a5 100644 --- a/keystone-moon/keystone/common/validation/__init__.py +++ b/keystone-moon/keystone/common/validation/__init__.py @@ -12,8 +12,11 @@ """Request body validating middleware for OpenStack Identity resources.""" import functools +import inspect from keystone.common.validation import validators +from keystone import exception +from keystone.i18n import _ def validated(request_body_schema, resource_to_validate): @@ -24,15 +27,47 @@ def validated(request_body_schema, resource_to_validate): :param request_body_schema: a schema to validate the resource reference :param resource_to_validate: the reference to validate + :raises keystone.exception.ValidationError: if `resource_to_validate` is + not passed by or passed with an empty value (see wrapper method + below). + :raises TypeError: at decoration time when the expected resource to + validate isn't found in the decorated method's + signature """ schema_validator = validators.SchemaValidator(request_body_schema) def add_validator(func): + argspec = inspect.getargspec(func) + try: + arg_index = argspec.args.index(resource_to_validate) + except ValueError: + raise TypeError(_('validated expected to find %(param_name)r in ' + 'function signature for %(func_name)r.') % + {'param_name': resource_to_validate, + 'func_name': func.__name__}) + @functools.wraps(func) def wrapper(*args, **kwargs): - if resource_to_validate in kwargs: + if kwargs.get(resource_to_validate): schema_validator.validate(kwargs[resource_to_validate]) + else: + try: + resource = args[arg_index] + # If resource to be validated is empty, no need to do + # validation since the message given by jsonschema doesn't + # help in this case. + if resource: + schema_validator.validate(resource) + else: + raise exception.ValidationError( + attribute=resource_to_validate, + target='request body') + # We cannot find the resource neither from kwargs nor args. + except IndexError: + raise exception.ValidationError( + attribute=resource_to_validate, + target='request body') return func(*args, **kwargs) return wrapper return add_validator diff --git a/keystone-moon/keystone/common/validation/parameter_types.py b/keystone-moon/keystone/common/validation/parameter_types.py index c5908836..1bc81383 100644 --- a/keystone-moon/keystone/common/validation/parameter_types.py +++ b/keystone-moon/keystone/common/validation/parameter_types.py @@ -28,6 +28,12 @@ name = { 'maxLength': 255 } +external_id_string = { + 'type': 'string', + 'minLength': 1, + 'maxLength': 64 +} + id_string = { 'type': 'string', 'minLength': 1, diff --git a/keystone-moon/keystone/common/wsgi.py b/keystone-moon/keystone/common/wsgi.py index 6ee8150d..0dee954b 100644 --- a/keystone-moon/keystone/common/wsgi.py +++ b/keystone-moon/keystone/common/wsgi.py @@ -20,7 +20,7 @@ import copy import itertools -import urllib +import wsgiref.util from oslo_config import cfg import oslo_i18n @@ -49,10 +49,12 @@ LOG = log.getLogger(__name__) # Environment variable used to pass the request context CONTEXT_ENV = 'openstack.context' - # Environment variable used to pass the request params PARAMS_ENV = 'openstack.params' +JSON_ENCODE_CONTENT_TYPES = set(['application/json', + 'application/json-home']) + def validate_token_bind(context, token_ref): bind_mode = CONF.token.enforce_token_bind @@ -84,7 +86,7 @@ def validate_token_bind(context, token_ref): LOG.info(_LI("Named bind mode %s not in bind information"), name) raise exception.Unauthorized() - for bind_type, identifier in six.iteritems(bind): + for bind_type, identifier in bind.items(): if bind_type == 'kerberos': if not (context['environment'].get('AUTH_TYPE', '').lower() == 'negotiate'): @@ -195,8 +197,16 @@ class Application(BaseApplication): # allow middleware up the stack to provide context, params and headers. context = req.environ.get(CONTEXT_ENV, {}) - context['query_string'] = dict(six.iteritems(req.params)) - context['headers'] = dict(six.iteritems(req.headers)) + + try: + context['query_string'] = dict(req.params.items()) + except UnicodeDecodeError as e: + # The webob package throws UnicodeError when a request cannot be + # decoded. Raise ValidationError instead to avoid an UnknownError. + msg = _('Query string is not UTF-8 encoded') + raise exception.ValidationError(msg) + + context['headers'] = dict(req.headers.items()) context['path'] = req.environ['PATH_INFO'] scheme = (None if not CONF.secure_proxy_ssl_header else req.environ.get(CONF.secure_proxy_ssl_header)) @@ -211,8 +221,8 @@ class Application(BaseApplication): context['host_url'] = req.host_url params = req.environ.get(PARAMS_ENV, {}) # authentication and authorization attributes are set as environment - # values by the container and processed by the pipeline. the complete - # set is not yet know. + # values by the container and processed by the pipeline. The complete + # set is not yet known. context['environment'] = req.environ context['accept_header'] = req.accept req.environ = None @@ -227,11 +237,10 @@ class Application(BaseApplication): # NOTE(morganfainberg): use the request method to normalize the # response code between GET and HEAD requests. The HTTP status should # be the same. - req_method = req.environ['REQUEST_METHOD'].upper() - LOG.info('%(req_method)s %(path)s?%(params)s', { - 'req_method': req_method, - 'path': context['path'], - 'params': urllib.urlencode(req.params)}) + LOG.info('%(req_method)s %(uri)s', { + 'req_method': req.environ['REQUEST_METHOD'].upper(), + 'uri': wsgiref.util.request_uri(req.environ), + }) params = self._normalize_dict(params) @@ -270,7 +279,7 @@ class Application(BaseApplication): response_code = self._get_response_code(req) return render_response(body=result, status=response_code, - method=req_method) + method=req.environ['REQUEST_METHOD']) def _get_response_code(self, req): req_method = req.environ['REQUEST_METHOD'] @@ -284,17 +293,21 @@ class Application(BaseApplication): return arg.replace(':', '_').replace('-', '_') def _normalize_dict(self, d): - return {self._normalize_arg(k): v for (k, v) in six.iteritems(d)} + return {self._normalize_arg(k): v for (k, v) in d.items()} def assert_admin(self, context): + """Ensure the user is an admin. + + :raises keystone.exception.Unauthorized: if a token could not be + found/authorized, a user is invalid, or a tenant is + invalid/not scoped. + :raises keystone.exception.Forbidden: if the user is not an admin and + does not have the admin role + + """ + if not context['is_admin']: - try: - user_token_ref = token_model.KeystoneToken( - token_id=context['token_id'], - token_data=self.token_provider_api.validate_token( - context['token_id'])) - except exception.TokenNotFound as e: - raise exception.Unauthorized(e) + user_token_ref = utils.get_token_ref(context) validate_token_bind(context, user_token_ref) creds = copy.deepcopy(user_token_ref.metadata) @@ -353,16 +366,7 @@ class Application(BaseApplication): LOG.debug(('will not lookup trust as the request auth token is ' 'either absent or it is the system admin token')) return None - - try: - token_data = self.token_provider_api.validate_token( - context['token_id']) - except exception.TokenNotFound: - LOG.warning(_LW('Invalid token in _get_trust_id_for_request')) - raise exception.Unauthorized() - - token_ref = token_model.KeystoneToken(token_id=context['token_id'], - token_data=token_data) + token_ref = utils.get_token_ref(context) return token_ref.trust_id @classmethod @@ -371,8 +375,7 @@ class Application(BaseApplication): if url: substitutions = dict( - itertools.chain(six.iteritems(CONF), - six.iteritems(CONF.eventlet_server))) + itertools.chain(CONF.items(), CONF.eventlet_server.items())) url = url % substitutions else: @@ -491,7 +494,7 @@ class Debug(Middleware): resp = req.get_response(self.application) if not hasattr(LOG, 'isEnabledFor') or LOG.isEnabledFor(LOG.debug): LOG.debug('%s %s %s', ('*' * 20), 'RESPONSE HEADERS', ('*' * 20)) - for (key, value) in six.iteritems(resp.headers): + for (key, value) in resp.headers.items(): LOG.debug('%s = %s', key, value) LOG.debug('') @@ -603,7 +606,7 @@ class ExtensionRouter(Router): mapper = routes.Mapper() self.application = application self.add_routes(mapper) - mapper.connect('{path_info:.*}', controller=self.application) + mapper.connect('/{path_info:.*}', controller=self.application) super(ExtensionRouter, self).__init__(mapper) def add_routes(self, mapper): @@ -657,7 +660,7 @@ class RoutersBase(object): get_action=None, head_action=None, get_head_action=None, put_action=None, post_action=None, patch_action=None, delete_action=None, get_post_action=None, - path_vars=None, status=None): + path_vars=None, status=json_home.Status.STABLE): if get_head_action: getattr(controller, get_head_action) # ensure the attribute exists mapper.connect(path, controller=controller, action=get_head_action, @@ -699,13 +702,7 @@ class RoutersBase(object): else: resource_data['href'] = path - if status: - if not json_home.Status.is_supported(status): - raise exception.Error(message=_( - 'Unexpected status requested for JSON Home response, %s') % - status) - resource_data.setdefault('hints', {}) - resource_data['hints']['status'] = status + json_home.Status.update_resource_data(resource_data, status) self.v3_resources.append((rel, resource_data)) @@ -762,8 +759,6 @@ def render_response(body=None, status=None, headers=None, method=None): else: content_type = None - JSON_ENCODE_CONTENT_TYPES = ('application/json', - 'application/json-home',) if content_type is None or content_type in JSON_ENCODE_CONTENT_TYPES: body = jsonutils.dumps(body, cls=utils.SmarterEncoder) if content_type is None: @@ -774,7 +769,7 @@ def render_response(body=None, status=None, headers=None, method=None): status='%s %s' % status, headerlist=headers) - if method == 'HEAD': + if method and method.upper() == 'HEAD': # NOTE(morganfainberg): HEAD requests should return the same status # as a GET request and same headers (including content-type and # content-length). The webob.Response object automatically changes @@ -785,7 +780,7 @@ def render_response(body=None, status=None, headers=None, method=None): # both py2x and py3x. stored_headers = resp.headers.copy() resp.body = b'' - for header, value in six.iteritems(stored_headers): + for header, value in stored_headers.items(): resp.headers[header] = value return resp @@ -820,8 +815,7 @@ def render_exception(error, context=None, request=None, user_locale=None): url = 'http://localhost:%d' % CONF.eventlet_server.public_port else: substitutions = dict( - itertools.chain(six.iteritems(CONF), - six.iteritems(CONF.eventlet_server))) + itertools.chain(CONF.items(), CONF.eventlet_server.items())) url = url % substitutions headers.append(('WWW-Authenticate', 'Keystone uri="%s"' % url)) |