From 92fd2dbfb672d7b2b1cdfd5dd5cf89f7716b3e12 Mon Sep 17 00:00:00 2001 From: asteroide Date: Tue, 1 Sep 2015 16:03:26 +0200 Subject: Update Keystone code from official Github repository with branch Master on 09/01/2015. Change-Id: I0ff6099e6e2580f87f502002a998bbfe12673498 --- keystone-moon/keystone/common/sql/core.py | 117 +++++++++++++-------- .../sql/migrate_repo/versions/045_placeholder.py | 4 - .../sql/migrate_repo/versions/046_placeholder.py | 4 - .../sql/migrate_repo/versions/047_placeholder.py | 4 - .../sql/migrate_repo/versions/048_placeholder.py | 4 - .../sql/migrate_repo/versions/049_placeholder.py | 4 - .../versions/050_fk_consistent_indexes.py | 14 +-- .../migrate_repo/versions/051_add_id_mapping.py | 8 -- .../versions/052_add_auth_url_to_region.py | 9 +- .../versions/053_endpoint_to_region_association.py | 66 ------------ .../versions/054_add_actor_id_index.py | 10 +- .../versions/055_add_indexes_to_token_table.py | 10 -- .../sql/migrate_repo/versions/056_placeholder.py | 4 - .../sql/migrate_repo/versions/057_placeholder.py | 4 - .../sql/migrate_repo/versions/058_placeholder.py | 4 - .../sql/migrate_repo/versions/059_placeholder.py | 4 - .../sql/migrate_repo/versions/060_placeholder.py | 4 - .../versions/061_add_parent_project.py | 15 +-- .../versions/062_drop_assignment_role_fk.py | 6 -- .../versions/063_drop_region_auth_url.py | 10 +- .../versions/064_drop_user_and_group_fk.py | 6 -- .../migrate_repo/versions/065_add_domain_config.py | 11 +- .../versions/066_fixup_service_name_value.py | 13 +-- .../versions/067_drop_redundant_mysql_index.py | 25 +++++ .../sql/migrate_repo/versions/068_placeholder.py | 18 ++++ .../sql/migrate_repo/versions/069_placeholder.py | 18 ++++ .../sql/migrate_repo/versions/070_placeholder.py | 18 ++++ .../sql/migrate_repo/versions/071_placeholder.py | 18 ++++ .../sql/migrate_repo/versions/072_placeholder.py | 18 ++++ .../versions/073_insert_assignment_inherited_pk.py | 114 ++++++++++++++++++++ .../versions/074_add_is_domain_project.py | 27 +++++ .../keystone/common/sql/migration_helpers.py | 66 +++--------- 32 files changed, 363 insertions(+), 294 deletions(-) create mode 100644 keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py create mode 100644 keystone-moon/keystone/common/sql/migrate_repo/versions/068_placeholder.py create mode 100644 keystone-moon/keystone/common/sql/migrate_repo/versions/069_placeholder.py create mode 100644 keystone-moon/keystone/common/sql/migrate_repo/versions/070_placeholder.py create mode 100644 keystone-moon/keystone/common/sql/migrate_repo/versions/071_placeholder.py create mode 100644 keystone-moon/keystone/common/sql/migrate_repo/versions/072_placeholder.py create mode 100644 keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py create mode 100644 keystone-moon/keystone/common/sql/migrate_repo/versions/074_add_is_domain_project.py (limited to 'keystone-moon/keystone/common/sql') diff --git a/keystone-moon/keystone/common/sql/core.py b/keystone-moon/keystone/common/sql/core.py index bf168701..ebd61bb7 100644 --- a/keystone-moon/keystone/common/sql/core.py +++ b/keystone-moon/keystone/common/sql/core.py @@ -239,6 +239,39 @@ def truncated(f): return wrapper +class _WontMatch(Exception): + """Raised to indicate that the filter won't match. + + This is raised to short-circuit the computation of the filter as soon as + it's discovered that the filter requested isn't going to match anything. + + A filter isn't going to match anything if the value is too long for the + field, for example. + + """ + + @classmethod + def check(cls, value, col_attr): + """Check if the value can match given the column attributes. + + Raises this class if the value provided can't match any value in the + column in the table given the column's attributes. For example, if the + column is a string and the value is longer than the column then it + won't match any value in the column in the table. + + """ + col = col_attr.property.columns[0] + if isinstance(col.type, sql.types.Boolean): + # The column is a Boolean, we should have already validated input. + return + if not col.type.length: + # The column doesn't have a length so can't validate anymore. + return + if len(value) > col.type.length: + raise cls() + # Otherwise the value could match a value in the column. + + def _filter(model, query, hints): """Applies filtering to a query. @@ -251,16 +284,14 @@ def _filter(model, query, hints): :returns query: query, updated with any filters satisfied """ - def inexact_filter(model, query, filter_, satisfied_filters, hints): + def inexact_filter(model, query, filter_, satisfied_filters): """Applies an inexact filter to a query. :param model: the table model in question :param query: query to apply filters to - :param filter_: the dict that describes this filter - :param satisfied_filters: a cumulative list of satisfied filters, to - which filter_ will be added if it is - satisfied. - :param hints: contains the list of filters yet to be satisfied. + :param dict filter_: describes this filter + :param list satisfied_filters: filter_ will be added if it is + satisfied. :returns query: query updated to add any inexact filters we could satisfy @@ -278,10 +309,13 @@ def _filter(model, query, hints): return query if filter_['comparator'] == 'contains': + _WontMatch.check(filter_['value'], column_attr) query_term = column_attr.ilike('%%%s%%' % filter_['value']) elif filter_['comparator'] == 'startswith': + _WontMatch.check(filter_['value'], column_attr) query_term = column_attr.ilike('%s%%' % filter_['value']) elif filter_['comparator'] == 'endswith': + _WontMatch.check(filter_['value'], column_attr) query_term = column_attr.ilike('%%%s' % filter_['value']) else: # It's a filter we don't understand, so let the caller @@ -291,53 +325,50 @@ def _filter(model, query, hints): satisfied_filters.append(filter_) return query.filter(query_term) - def exact_filter( - model, filter_, satisfied_filters, cumulative_filter_dict, hints): + def exact_filter(model, filter_, cumulative_filter_dict): """Applies an exact filter to a query. :param model: the table model in question - :param filter_: the dict that describes this filter - :param satisfied_filters: a cumulative list of satisfied filters, to - which filter_ will be added if it is - satisfied. - :param cumulative_filter_dict: a dict that describes the set of - exact filters built up so far - :param hints: contains the list of filters yet to be satisfied. - - :returns: updated cumulative dict + :param dict filter_: describes this filter + :param dict cumulative_filter_dict: describes the set of exact filters + built up so far """ key = filter_['name'] - if isinstance(getattr(model, key).property.columns[0].type, - sql.types.Boolean): + + col = getattr(model, key) + if isinstance(col.property.columns[0].type, sql.types.Boolean): cumulative_filter_dict[key] = ( utils.attr_as_boolean(filter_['value'])) else: + _WontMatch.check(filter_['value'], col) cumulative_filter_dict[key] = filter_['value'] - satisfied_filters.append(filter_) - return cumulative_filter_dict - - filter_dict = {} - satisfied_filters = [] - for filter_ in hints.filters: - if filter_['name'] not in model.attributes: - continue - if filter_['comparator'] == 'equals': - filter_dict = exact_filter( - model, filter_, satisfied_filters, filter_dict, hints) - else: - query = inexact_filter( - model, query, filter_, satisfied_filters, hints) - # Apply any exact filters we built up - if filter_dict: - query = query.filter_by(**filter_dict) + try: + filter_dict = {} + satisfied_filters = [] + for filter_ in hints.filters: + if filter_['name'] not in model.attributes: + continue + if filter_['comparator'] == 'equals': + exact_filter(model, filter_, filter_dict) + satisfied_filters.append(filter_) + else: + query = inexact_filter(model, query, filter_, + satisfied_filters) + + # Apply any exact filters we built up + if filter_dict: + query = query.filter_by(**filter_dict) + + # Remove satisfied filters, then the caller will know remaining filters + for filter_ in satisfied_filters: + hints.filters.remove(filter_) - # Remove satisfied filters, then the caller will know remaining filters - for filter_ in satisfied_filters: - hints.filters.remove(filter_) - - return query + return query + except _WontMatch: + hints.cannot_match = True + return def _limit(query, hints): @@ -378,6 +409,10 @@ def filter_limit_query(model, query, hints): # First try and satisfy any filters query = _filter(model, query, hints) + if hints.cannot_match: + # Nothing's going to match, so don't bother with the query. + return [] + # NOTE(henry-nash): Any unsatisfied filters will have been left in # the hints list for the controller to handle. We can only try and # limit here if all the filters are already satisfied since, if not, diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py index b6f40719..2a98fb90 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py @@ -19,7 +19,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py index 535a0944..c4b41580 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py @@ -27,7 +27,8 @@ def upgrade(migrate_engine): # names, depending on version of MySQL used. We shoud make this naming # consistent, by reverting index name to a consistent condition. if any(i for i in endpoint.indexes if - i.columns.keys() == ['service_id'] and i.name != 'service_id'): + list(i.columns.keys()) == ['service_id'] + and i.name != 'service_id'): # NOTE(i159): by this action will be made re-creation of an index # with the new name. This can be considered as renaming under the # MySQL rules. @@ -37,13 +38,6 @@ def upgrade(migrate_engine): meta, autoload=True) if any(i for i in user_group_membership.indexes if - i.columns.keys() == ['group_id'] and i.name != 'group_id'): + list(i.columns.keys()) == ['group_id'] + and i.name != 'group_id'): sa.Index('group_id', user_group_membership.c.group_id).create() - - -def downgrade(migrate_engine): - # NOTE(i159): index exists only in MySQL schemas, and got an inconsistent - # name only when MySQL 5.5 renamed it after re-creation - # (during migrations). So we just fixed inconsistency, there is no - # necessity to revert it. - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py index 074fbb63..59720f6e 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py @@ -39,11 +39,3 @@ def upgrade(migrate_engine): mysql_engine='InnoDB', mysql_charset='utf8') mapping_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - assignment = sql.Table(MAPPING_TABLE, meta, autoload=True) - assignment.drop(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py index 9f1fd9f0..86302a8f 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py @@ -14,6 +14,7 @@ import sqlalchemy as sql + _REGION_TABLE_NAME = 'region' @@ -24,11 +25,3 @@ def upgrade(migrate_engine): region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) url_column = sql.Column('url', sql.String(255), nullable=True) region_table.create_column(url_column) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) - region_table.drop_column('url') diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py index 6dc0004f..c2be48f4 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py @@ -12,7 +12,6 @@ # License for the specific language governing permissions and limitations # under the License. - """Migrated the endpoint 'region' column to 'region_id. In addition to the rename, the new column is made a foreign key to the @@ -36,25 +35,9 @@ b. For each endpoint ii. Assign the id to the region_id column c. Remove the column region - -To Downgrade: - -Endpoint Table - -a. Add back in the region column -b. For each endpoint - i. Copy the region_id column to the region column -c. Remove the column region_id - -Region Table - -Decrease the size of the id column in the region table, making sure that -we don't get classing primary keys. - """ import migrate -import six import sqlalchemy as sql from sqlalchemy.orm import sessionmaker @@ -90,39 +73,6 @@ def _migrate_to_region_id(migrate_engine, region_table, endpoint_table): name='fk_endpoint_region_id').create() -def _migrate_to_region(migrate_engine, region_table, endpoint_table): - endpoints = list(endpoint_table.select().execute()) - - for endpoint in endpoints: - new_values = {'region': endpoint.region_id} - f = endpoint_table.c.id == endpoint.id - update = endpoint_table.update().where(f).values(new_values) - migrate_engine.execute(update) - - if 'sqlite' != migrate_engine.name: - migrate.ForeignKeyConstraint( - columns=[endpoint_table.c.region_id], - refcolumns=[region_table.c.id], - name='fk_endpoint_region_id').drop() - endpoint_table.c.region_id.drop() - - -def _prepare_regions_for_id_truncation(migrate_engine, region_table): - """Ensure there are no IDs that are bigger than 64 chars. - - The size of the id and parent_id fields where increased from 64 to 255 - during the upgrade. On downgrade we have to make sure that the ids can - fit in the new column size. For rows with ids greater than this, we have - no choice but to dump them. - - """ - for region in list(region_table.select().execute()): - if (len(six.text_type(region.id)) > 64 or - len(six.text_type(region.parent_region_id)) > 64): - delete = region_table.delete(region_table.c.id == region.id) - migrate_engine.execute(delete) - - def upgrade(migrate_engine): meta = sql.MetaData() meta.bind = migrate_engine @@ -138,19 +88,3 @@ def upgrade(migrate_engine): _migrate_to_region_id(migrate_engine, region_table, endpoint_table) endpoint_table.c.region.drop() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table('region', meta, autoload=True) - endpoint_table = sql.Table('endpoint', meta, autoload=True) - region_column = sql.Column('region', sql.String(length=255)) - region_column.create(endpoint_table) - - _migrate_to_region(migrate_engine, region_table, endpoint_table) - _prepare_regions_for_id_truncation(migrate_engine, region_table) - - region_table.c.id.alter(type=sql.String(length=64)) - region_table.c.parent_region_id.alter(type=sql.String(length=64)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py index 33b13b7d..caf4d66f 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py @@ -14,6 +14,7 @@ import sqlalchemy as sql + ASSIGNMENT_TABLE = 'assignment' @@ -24,12 +25,3 @@ def upgrade(migrate_engine): assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True) idx = sql.Index('ix_actor_id', assignment.c.actor_id) idx.create(migrate_engine) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True) - idx = sql.Index('ix_actor_id', assignment.c.actor_id) - idx.drop(migrate_engine) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py index 1cfddd3f..a7f327ea 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py @@ -23,13 +23,3 @@ def upgrade(migrate_engine): sql.Index('ix_token_user_id', token.c.user_id).create() sql.Index('ix_token_trust_id', token.c.trust_id).create() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - token = sql.Table('token', meta, autoload=True) - - sql.Index('ix_token_user_id', token.c.user_id).drop() - sql.Index('ix_token_trust_id', token.c.trust_id).drop() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py index 5f82254f..8bb40490 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py @@ -16,7 +16,3 @@ def upgrade(migrate_engine): pass - - -def downgrade(migration_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py index bb8ef9f6..ca9b3ce2 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py @@ -14,6 +14,7 @@ import sqlalchemy as sql from keystone.common.sql import migration_helpers + _PROJECT_TABLE_NAME = 'project' _PARENT_ID_COLUMN_NAME = 'parent_id' @@ -38,17 +39,3 @@ def upgrade(migrate_engine): if migrate_engine.name == 'sqlite': return migration_helpers.add_constraints(list_constraints(project_table)) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) - - # SQLite does not support constraints, and querying the constraints - # raises an exception - if migrate_engine.name != 'sqlite': - migration_helpers.remove_constraints(list_constraints(project_table)) - - project_table.drop_column(_PARENT_ID_COLUMN_NAME) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py index 5a33486c..f7a69bb6 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py @@ -33,9 +33,3 @@ def upgrade(migrate_engine): if migrate_engine.name == 'sqlite': return migration_helpers.remove_constraints(list_constraints(migrate_engine)) - - -def downgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - return - migration_helpers.add_constraints(list_constraints(migrate_engine)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py index 109a8412..e45133ab 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py @@ -12,6 +12,7 @@ import sqlalchemy as sql + _REGION_TABLE_NAME = 'region' @@ -21,12 +22,3 @@ def upgrade(migrate_engine): region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) region_table.drop_column('url') - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) - url_column = sql.Column('url', sql.String(255), nullable=True) - region_table.create_column(url_column) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py index bca00902..637f2151 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py @@ -37,9 +37,3 @@ def upgrade(migrate_engine): if migrate_engine.name == 'sqlite': return migration_helpers.remove_constraints(list_constraints(migrate_engine)) - - -def downgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - return - migration_helpers.add_constraints(list_constraints(migrate_engine)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py index fd8717d2..63a86c11 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py @@ -14,6 +14,7 @@ import sqlalchemy as sql from keystone.common import sql as ks_sql + WHITELIST_TABLE = 'whitelisted_config' SENSITIVE_TABLE = 'sensitive_config' @@ -43,13 +44,3 @@ def upgrade(migrate_engine): mysql_engine='InnoDB', mysql_charset='utf8') sensitive_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - table = sql.Table(WHITELIST_TABLE, meta, autoload=True) - table.drop(migrate_engine, checkfirst=True) - table = sql.Table(SENSITIVE_TABLE, meta, autoload=True) - table.drop(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py index 3feadc53..fe0cee88 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py @@ -22,7 +22,11 @@ def upgrade(migrate_engine): services = list(service_table.select().execute()) for service in services: - extra_dict = jsonutils.loads(service.extra) + if service.extra is not None: + extra_dict = jsonutils.loads(service.extra) + else: + extra_dict = {} + # Skip records where service is not null if extra_dict.get('name') is not None: continue @@ -34,10 +38,3 @@ def upgrade(migrate_engine): f = service_table.c.id == service.id update = service_table.update().where(f).values(new_values) migrate_engine.execute(update) - - -def downgrade(migration_engine): - # The upgrade fixes the data inconsistency for the service name, - # it defaults the value to empty string. There is no necessity - # to revert it. - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py new file mode 100644 index 00000000..b9df1a55 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy + + +def upgrade(migrate_engine): + # NOTE(viktors): Migration 062 removed FK from `assignment` table, but + # MySQL silently creates indexes on FK constraints, so we should remove + # this index manually. + if migrate_engine.name == 'mysql': + meta = sqlalchemy.MetaData(bind=migrate_engine) + table = sqlalchemy.Table('assignment', meta, autoload=True) + for index in table.indexes: + if [c.name for c in index.columns] == ['role_id']: + index.drop(migrate_engine) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/068_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/068_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/068_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/069_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/069_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/069_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/070_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/070_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/070_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/071_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/071_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/071_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/072_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/072_placeholder.py new file mode 100644 index 00000000..111df9d4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/072_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Kilo backports. Do not use this number for new +# Liberty work. New Liberty work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py new file mode 100644 index 00000000..ffa210c4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py @@ -0,0 +1,114 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy as sql +from sqlalchemy.orm import sessionmaker + +from keystone.assignment.backends import sql as assignment_sql + + +def upgrade(migrate_engine): + """Inserts inherited column to assignment table PK contraints. + + For non-SQLite databases, it changes the constraint in the existing table. + + For SQLite, since changing constraints is not supported, it recreates the + assignment table with the new PK constraint and migrates the existing data. + + """ + + ASSIGNMENT_TABLE_NAME = 'assignment' + + metadata = sql.MetaData() + metadata.bind = migrate_engine + + # Retrieve the existing assignment table + assignment_table = sql.Table(ASSIGNMENT_TABLE_NAME, metadata, + autoload=True) + + if migrate_engine.name == 'sqlite': + ACTOR_ID_INDEX_NAME = 'ix_actor_id' + TMP_ASSIGNMENT_TABLE_NAME = 'tmp_assignment' + + # Define the new assignment table with a temporary name + new_assignment_table = sql.Table( + TMP_ASSIGNMENT_TABLE_NAME, metadata, + sql.Column('type', sql.Enum( + assignment_sql.AssignmentType.USER_PROJECT, + assignment_sql.AssignmentType.GROUP_PROJECT, + assignment_sql.AssignmentType.USER_DOMAIN, + assignment_sql.AssignmentType.GROUP_DOMAIN, + name='type'), + nullable=False), + sql.Column('actor_id', sql.String(64), nullable=False), + sql.Column('target_id', sql.String(64), nullable=False), + sql.Column('role_id', sql.String(64), sql.ForeignKey('role.id'), + nullable=False), + sql.Column('inherited', sql.Boolean, default=False, + nullable=False), + sql.PrimaryKeyConstraint('type', 'actor_id', 'target_id', + 'role_id', 'inherited'), + mysql_engine='InnoDB', + mysql_charset='utf8') + + # Create the new assignment table + new_assignment_table.create(migrate_engine, checkfirst=True) + + # Change the index from the existing assignment table to the new one + sql.Index(ACTOR_ID_INDEX_NAME, assignment_table.c.actor_id).drop() + sql.Index(ACTOR_ID_INDEX_NAME, + new_assignment_table.c.actor_id).create() + + # Instantiate session + maker = sessionmaker(bind=migrate_engine) + session = maker() + + # Migrate existing data + insert = new_assignment_table.insert().from_select( + assignment_table.c, select=session.query(assignment_table)) + session.execute(insert) + session.commit() + + # Drop the existing assignment table, in favor of the new one + assignment_table.deregister() + assignment_table.drop() + + # Finally, rename the new table to the original assignment table name + new_assignment_table.rename(ASSIGNMENT_TABLE_NAME) + elif migrate_engine.name == 'ibm_db_sa': + # Recreate the existing constraint, marking the inherited column as PK + # for DB2. + + # This is a workaround to the general case in the else statement below. + # Due to a bug in the DB2 sqlalchemy dialect, Column.alter() actually + # creates a primary key over only the "inherited" column. This is wrong + # because the primary key for the table actually covers other columns + # too, not just the "inherited" column. Since the primary key already + # exists for the table after the Column.alter() call, it causes the + # next line to fail with an error that the primary key already exists. + + # The workaround here skips doing the Column.alter(). This causes a + # warning message since the metadata is out of sync. We can remove this + # workaround once the DB2 sqlalchemy dialect is fixed. + # DB2 Issue: https://code.google.com/p/ibm-db/issues/detail?id=173 + + migrate.PrimaryKeyConstraint(table=assignment_table).drop() + migrate.PrimaryKeyConstraint( + assignment_table.c.type, assignment_table.c.actor_id, + assignment_table.c.target_id, assignment_table.c.role_id, + assignment_table.c.inherited).create() + else: + # Recreate the existing constraint, marking the inherited column as PK + migrate.PrimaryKeyConstraint(table=assignment_table).drop() + assignment_table.c.inherited.alter(primary_key=True) + migrate.PrimaryKeyConstraint(table=assignment_table).create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/074_add_is_domain_project.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/074_add_is_domain_project.py new file mode 100644 index 00000000..dcb89b07 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/074_add_is_domain_project.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + + +_PROJECT_TABLE_NAME = 'project' +_IS_DOMAIN_COLUMN_NAME = 'is_domain' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) + is_domain = sql.Column(_IS_DOMAIN_COLUMN_NAME, sql.Boolean, nullable=False, + server_default='0', default=False) + project_table.create_column(is_domain) diff --git a/keystone-moon/keystone/common/sql/migration_helpers.py b/keystone-moon/keystone/common/sql/migration_helpers.py index 86932995..aaa59f70 100644 --- a/keystone-moon/keystone/common/sql/migration_helpers.py +++ b/keystone-moon/keystone/common/sql/migration_helpers.py @@ -143,37 +143,21 @@ def _sync_common_repo(version): abs_path = find_migrate_repo() init_version = migrate_repo.DB_INIT_VERSION engine = sql.get_engine() + _assert_not_schema_downgrade(version=version) migration.db_sync(engine, abs_path, version=version, - init_version=init_version) + init_version=init_version, sanity_check=False) -def _fix_federation_tables(engine): - """Fix the identity_provider, federation_protocol and mapping tables - to be InnoDB and Charset UTF8. - - This function is to work around bug #1426334. This has occurred because - the original migration did not specify InnoDB and charset utf8. Due - to the sanity_check, a deployer can get wedged here and require manual - database changes to fix. - """ - # NOTE(marco-fargetta) This is a workaround to "fix" that tables only - # if we're under MySQL - if engine.name == 'mysql': - # * Disable any check for the foreign keys because they prevent the - # alter table to execute - engine.execute("SET foreign_key_checks = 0") - # * Make the tables using InnoDB engine - engine.execute("ALTER TABLE identity_provider Engine=InnoDB") - engine.execute("ALTER TABLE federation_protocol Engine=InnoDB") - engine.execute("ALTER TABLE mapping Engine=InnoDB") - # * Make the tables using utf8 encoding - engine.execute("ALTER TABLE identity_provider " - "CONVERT TO CHARACTER SET utf8") - engine.execute("ALTER TABLE federation_protocol " - "CONVERT TO CHARACTER SET utf8") - engine.execute("ALTER TABLE mapping CONVERT TO CHARACTER SET utf8") - # * Revert the foreign keys check back - engine.execute("SET foreign_key_checks = 1") +def _assert_not_schema_downgrade(extension=None, version=None): + if version is not None: + try: + current_ver = int(six.text_type(get_db_version(extension))) + if int(version) < current_ver: + raise migration.exception.DbMigrationError() + except exceptions.DatabaseNotControlledError: + # NOTE(morganfainberg): The database is not controlled, this action + # cannot be a downgrade. + pass def _sync_extension_repo(extension, version): @@ -198,27 +182,11 @@ def _sync_extension_repo(extension, version): except exception.MigrationNotProvided as e: print(e) sys.exit(1) - try: - migration.db_sync(engine, abs_path, version=version, - init_version=init_version) - except ValueError: - # NOTE(marco-fargetta): ValueError is raised from the sanity check ( - # verifies that tables are utf8 under mysql). The federation_protocol, - # identity_provider and mapping tables were not initially built with - # InnoDB and utf8 as part of the table arguments when the migration - # was initially created. Bug #1426334 is a scenario where the deployer - # can get wedged, unable to upgrade or downgrade. - # This is a workaround to "fix" those tables if we're under MySQL and - # the version is before the 6 because before the tables were introduced - # before and patched when migration 5 was available - if engine.name == 'mysql' and \ - int(six.text_type(get_db_version(extension))) < 6: - _fix_federation_tables(engine) - # The migration is applied again after the fix - migration.db_sync(engine, abs_path, version=version, - init_version=init_version) - else: - raise + + _assert_not_schema_downgrade(extension=extension, version=version) + + migration.db_sync(engine, abs_path, version=version, + init_version=init_version, sanity_check=False) def sync_database_to_version(extension=None, version=None): -- cgit 1.2.3-korg