diff options
| author |   Ruan HE <ruan.he@orange.com> | 2016-06-09 08:12:34 +0000 |
|---|---|---|
| committer |   Gerrit Code Review <gerrit@172.30.200.206> | 2016-06-09 08:12:34 +0000 |
| commit | 4bc079a2664f9a407e332291f34d174625a9d5ea (patch) | |
| tree | 7481cd5d0a9b3ce37c44c797a1e0d39881221cbe /keystone-moon/keystone/common/sql | |
| parent | 2f179c5790fbbf6144205d3c6e5089e6eb5f048a (diff) | |
| parent | 2e7b4f2027a1147ca28301e4f88adf8274b39a1f (diff) | |
Merge "Update Keystone core to Mitaka."
Diffstat (limited to 'keystone-moon/keystone/common/sql')
45 files changed, 1185 insertions, 796 deletions
diff --git a/keystone-moon/keystone/common/sql/core.py b/keystone-moon/keystone/common/sql/core.py index ebd61bb7..cb026356 100644 --- a/keystone-moon/keystone/common/sql/core.py +++ b/keystone-moon/keystone/common/sql/core.py @@ -18,14 +18,13 @@ Before using this module, call initialize(). This has to be done before CONF() because it sets up configuration options. """ -import contextlib import functools from oslo_config import cfg from oslo_db import exception as db_exception from oslo_db import options as db_options +from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import models -from oslo_db.sqlalchemy import session as db_session from oslo_log import log from oslo_serialization import jsonutils import six @@ -34,6 +33,7 @@ from sqlalchemy.ext import declarative from sqlalchemy.orm.attributes import flag_modified, InstrumentedAttribute from sqlalchemy import types as sql_types +from keystone.common import driver_hints from keystone.common import utils from keystone import exception from keystone.i18n import _ @@ -68,7 +68,6 @@ flag_modified = flag_modified def initialize(): """Initialize the module.""" - db_options.set_defaults( CONF, connection="sqlite:///keystone.db") @@ -166,77 +165,47 @@ class ModelDictMixin(object): return {name: getattr(self, name) for name in names} -_engine_facade = None +_main_context_manager = None -def _get_engine_facade(): - global _engine_facade +def _get_main_context_manager(): + global _main_context_manager - if not _engine_facade: - _engine_facade = db_session.EngineFacade.from_config(CONF) + if not _main_context_manager: + _main_context_manager = enginefacade.transaction_context() - return _engine_facade + return _main_context_manager def cleanup(): - global _engine_facade + global _main_context_manager - _engine_facade = None + _main_context_manager = None -def get_engine(): - return _get_engine_facade().get_engine() +_CONTEXT = None -def get_session(expire_on_commit=False): - return _get_engine_facade().get_session(expire_on_commit=expire_on_commit) +def _get_context(): + global _CONTEXT + if _CONTEXT is None: + # NOTE(dims): Delay the `threading.local` import to allow for + # eventlet/gevent monkeypatching to happen + import threading + _CONTEXT = threading.local() + return _CONTEXT -@contextlib.contextmanager -def transaction(expire_on_commit=False): - """Return a SQLAlchemy session in a scoped transaction.""" - session = get_session(expire_on_commit=expire_on_commit) - with session.begin(): - yield session +def session_for_read(): + return _get_main_context_manager().reader.using(_get_context()) -def truncated(f): - """Ensure list truncation is detected in Driver list entity methods. +def session_for_write(): + return _get_main_context_manager().writer.using(_get_context()) - This is designed to wrap and sql Driver list_{entity} methods in order to - calculate if the resultant list has been truncated. Provided a limit dict - is found in the hints list, we increment the limit by one so as to ask the - wrapped function for one more entity than the limit, and then once the list - has been generated, we check to see if the original limit has been - exceeded, in which case we truncate back to that limit and set the - 'truncated' boolean to 'true' in the hints limit dict. - """ - @functools.wraps(f) - def wrapper(self, hints, *args, **kwargs): - if not hasattr(hints, 'limit'): - raise exception.UnexpectedError( - _('Cannot truncate a driver call without hints list as ' - 'first parameter after self ')) - - if hints.limit is None: - return f(self, hints, *args, **kwargs) - - # A limit is set, so ask for one more entry than we need - list_limit = hints.limit['limit'] - hints.set_limit(list_limit + 1) - ref_list = f(self, hints, *args, **kwargs) - - # If we got more than the original limit then trim back the list and - # mark it truncated. In both cases, make sure we set the limit back - # to its original value. - if len(ref_list) > list_limit: - hints.set_limit(list_limit, truncated=True) - return ref_list[:list_limit] - else: - hints.set_limit(list_limit) - return ref_list - return wrapper +def truncated(f): + return driver_hints.truncated(f) class _WontMatch(Exception): @@ -325,42 +294,41 @@ def _filter(model, query, hints): satisfied_filters.append(filter_) return query.filter(query_term) - def exact_filter(model, filter_, cumulative_filter_dict): + def exact_filter(model, query, filter_, satisfied_filters): """Applies an exact filter to a query. :param model: the table model in question + :param query: query to apply filters to :param dict filter_: describes this filter - :param dict cumulative_filter_dict: describes the set of exact filters - built up so far - + :param list satisfied_filters: filter_ will be added if it is + satisfied. + :returns query: query updated to add any exact filters we could + satisfy """ key = filter_['name'] col = getattr(model, key) if isinstance(col.property.columns[0].type, sql.types.Boolean): - cumulative_filter_dict[key] = ( - utils.attr_as_boolean(filter_['value'])) + filter_val = utils.attr_as_boolean(filter_['value']) else: _WontMatch.check(filter_['value'], col) - cumulative_filter_dict[key] = filter_['value'] + filter_val = filter_['value'] + + satisfied_filters.append(filter_) + return query.filter(col == filter_val) try: - filter_dict = {} satisfied_filters = [] for filter_ in hints.filters: if filter_['name'] not in model.attributes: continue if filter_['comparator'] == 'equals': - exact_filter(model, filter_, filter_dict) - satisfied_filters.append(filter_) + query = exact_filter(model, query, filter_, + satisfied_filters) else: query = inexact_filter(model, query, filter_, satisfied_filters) - # Apply any exact filters we built up - if filter_dict: - query = query.filter_by(**filter_dict) - # Remove satisfied filters, then the caller will know remaining filters for filter_ in satisfied_filters: hints.filters.remove(filter_) @@ -377,7 +345,7 @@ def _limit(query, hints): :param query: query to apply filters to :param hints: contains the list of filters and limit details. - :returns updated query + :returns: updated query """ # NOTE(henry-nash): If we were to implement pagination, then we diff --git a/keystone-moon/keystone/common/sql/migrate_repo/README b/keystone-moon/keystone/common/sql/migrate_repo/README index 6218f8ca..4ea8dd4f 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/README +++ b/keystone-moon/keystone/common/sql/migrate_repo/README @@ -1,4 +1,4 @@ This is a database migration repository. More information at -http://code.google.com/p/sqlalchemy-migrate/ +https://git.openstack.org/cgit/openstack/sqlalchemy-migrate diff --git a/keystone-moon/keystone/common/sql/migrate_repo/__init__.py b/keystone-moon/keystone/common/sql/migrate_repo/__init__.py index f73dfc12..e69de29b 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/__init__.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/__init__.py @@ -1,17 +0,0 @@ -# Copyright 2014 Mirantis.inc -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -DB_INIT_VERSION = 43 diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py deleted file mode 100644 index 2a98fb90..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Icehouse backports. Do not use this number for new -# Juno work. New Juno work starts after all the placeholders. -# -# See blueprint reserved-db-migrations-icehouse and the related discussion: -# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html - - -def upgrade(migrate_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py deleted file mode 100644 index 2a98fb90..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Icehouse backports. Do not use this number for new -# Juno work. New Juno work starts after all the placeholders. -# -# See blueprint reserved-db-migrations-icehouse and the related discussion: -# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html - - -def upgrade(migrate_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py deleted file mode 100644 index 2a98fb90..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Icehouse backports. Do not use this number for new -# Juno work. New Juno work starts after all the placeholders. -# -# See blueprint reserved-db-migrations-icehouse and the related discussion: -# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html - - -def upgrade(migrate_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py deleted file mode 100644 index 2a98fb90..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Icehouse backports. Do not use this number for new -# Juno work. New Juno work starts after all the placeholders. -# -# See blueprint reserved-db-migrations-icehouse and the related discussion: -# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html - - -def upgrade(migrate_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py deleted file mode 100644 index c4b41580..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2014 Mirantis.inc -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sa - - -def upgrade(migrate_engine): - - if migrate_engine.name == 'mysql': - meta = sa.MetaData(bind=migrate_engine) - endpoint = sa.Table('endpoint', meta, autoload=True) - - # NOTE(i159): MySQL requires indexes on referencing columns, and those - # indexes create automatically. That those indexes will have different - # names, depending on version of MySQL used. We shoud make this naming - # consistent, by reverting index name to a consistent condition. - if any(i for i in endpoint.indexes if - list(i.columns.keys()) == ['service_id'] - and i.name != 'service_id'): - # NOTE(i159): by this action will be made re-creation of an index - # with the new name. This can be considered as renaming under the - # MySQL rules. - sa.Index('service_id', endpoint.c.service_id).create() - - user_group_membership = sa.Table('user_group_membership', - meta, autoload=True) - - if any(i for i in user_group_membership.indexes if - list(i.columns.keys()) == ['group_id'] - and i.name != 'group_id'): - sa.Index('group_id', user_group_membership.c.group_id).create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py deleted file mode 100644 index 59720f6e..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2014 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.identity.mapping_backends import mapping - - -MAPPING_TABLE = 'id_mapping' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - mapping_table = sql.Table( - MAPPING_TABLE, - meta, - sql.Column('public_id', sql.String(64), primary_key=True), - sql.Column('domain_id', sql.String(64), nullable=False), - sql.Column('local_id', sql.String(64), nullable=False), - sql.Column('entity_type', sql.Enum( - mapping.EntityType.USER, - mapping.EntityType.GROUP, - name='entity_type'), - nullable=False), - sql.UniqueConstraint('domain_id', 'local_id', 'entity_type'), - mysql_engine='InnoDB', - mysql_charset='utf8') - mapping_table.create(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py deleted file mode 100644 index 86302a8f..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2014 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -_REGION_TABLE_NAME = 'region' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) - url_column = sql.Column('url', sql.String(255), nullable=True) - region_table.create_column(url_column) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py deleted file mode 100644 index c2be48f4..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Migrated the endpoint 'region' column to 'region_id. - -In addition to the rename, the new column is made a foreign key to the -respective 'region' in the region table, ensuring that we auto-create -any regions that are missing. Further, since the old region column -was 255 chars, and the id column in the region table is 64 chars, the size -of the id column in the region table is increased to match. - -To Upgrade: - - -Region Table - -Increase the size of the if column in the region table - -Endpoint Table - -a. Add the endpoint region_id column, that is a foreign key to the region table -b. For each endpoint - i. Ensure there is matching region in region table, and if not, create it - ii. Assign the id to the region_id column -c. Remove the column region - -""" - -import migrate -import sqlalchemy as sql -from sqlalchemy.orm import sessionmaker - - -def _migrate_to_region_id(migrate_engine, region_table, endpoint_table): - endpoints = list(endpoint_table.select().execute()) - - for endpoint in endpoints: - if endpoint.region is None: - continue - - region = list(region_table.select( - whereclause=region_table.c.id == endpoint.region).execute()) - if len(region) == 1: - region_id = region[0].id - else: - region_id = endpoint.region - region = {'id': region_id, - 'description': '', - 'extra': '{}'} - session = sessionmaker(bind=migrate_engine)() - region_table.insert(region).execute() - session.commit() - - new_values = {'region_id': region_id} - f = endpoint_table.c.id == endpoint.id - update = endpoint_table.update().where(f).values(new_values) - migrate_engine.execute(update) - - migrate.ForeignKeyConstraint( - columns=[endpoint_table.c.region_id], - refcolumns=[region_table.c.id], - name='fk_endpoint_region_id').create() - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table('region', meta, autoload=True) - region_table.c.id.alter(type=sql.String(length=255)) - region_table.c.parent_region_id.alter(type=sql.String(length=255)) - endpoint_table = sql.Table('endpoint', meta, autoload=True) - region_id_column = sql.Column('region_id', - sql.String(length=255), nullable=True) - region_id_column.create(endpoint_table) - - _migrate_to_region_id(migrate_engine, region_table, endpoint_table) - - endpoint_table.c.region.drop() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py deleted file mode 100644 index caf4d66f..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2014 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -ASSIGNMENT_TABLE = 'assignment' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True) - idx = sql.Index('ix_actor_id', assignment.c.actor_id) - idx.create(migrate_engine) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py deleted file mode 100644 index a7f327ea..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Add indexes to `user_id` and `trust_id` columns for the `token` table.""" - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - token = sql.Table('token', meta, autoload=True) - - sql.Index('ix_token_user_id', token.c.user_id).create() - sql.Index('ix_token_trust_id', token.c.trust_id).create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py deleted file mode 100644 index 8bb40490..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Juno backports. Do not use this number for new -# Kilo work. New Kilo work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py deleted file mode 100644 index ca9b3ce2..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common.sql import migration_helpers - - -_PROJECT_TABLE_NAME = 'project' -_PARENT_ID_COLUMN_NAME = 'parent_id' - - -def list_constraints(project_table): - constraints = [{'table': project_table, - 'fk_column': _PARENT_ID_COLUMN_NAME, - 'ref_column': project_table.c.id}] - - return constraints - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) - parent_id = sql.Column(_PARENT_ID_COLUMN_NAME, sql.String(64), - nullable=True) - project_table.create_column(parent_id) - - if migrate_engine.name == 'sqlite': - return - migration_helpers.add_constraints(list_constraints(project_table)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py deleted file mode 100644 index f7a69bb6..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy - -from keystone.common.sql import migration_helpers - - -def list_constraints(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - assignment_table = sqlalchemy.Table('assignment', meta, autoload=True) - role_table = sqlalchemy.Table('role', meta, autoload=True) - - constraints = [{'table': assignment_table, - 'fk_column': 'role_id', - 'ref_column': role_table.c.id}] - return constraints - - -def upgrade(migrate_engine): - # SQLite does not support constraints, and querying the constraints - # raises an exception - if migrate_engine.name == 'sqlite': - return - migration_helpers.remove_constraints(list_constraints(migrate_engine)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py deleted file mode 100644 index e45133ab..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -_REGION_TABLE_NAME = 'region' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) - region_table.drop_column('url') diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py deleted file mode 100644 index 637f2151..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy - -from keystone.common.sql import migration_helpers - - -def list_constraints(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - user_table = sqlalchemy.Table('user', meta, autoload=True) - group_table = sqlalchemy.Table('group', meta, autoload=True) - domain_table = sqlalchemy.Table('domain', meta, autoload=True) - - constraints = [{'table': user_table, - 'fk_column': 'domain_id', - 'ref_column': domain_table.c.id}, - {'table': group_table, - 'fk_column': 'domain_id', - 'ref_column': domain_table.c.id}] - return constraints - - -def upgrade(migrate_engine): - # SQLite does not support constraints, and querying the constraints - # raises an exception - if migrate_engine.name == 'sqlite': - return - migration_helpers.remove_constraints(list_constraints(migrate_engine)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py deleted file mode 100644 index 63a86c11..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common import sql as ks_sql - - -WHITELIST_TABLE = 'whitelisted_config' -SENSITIVE_TABLE = 'sensitive_config' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - whitelist_table = sql.Table( - WHITELIST_TABLE, - meta, - sql.Column('domain_id', sql.String(64), primary_key=True), - sql.Column('group', sql.String(255), primary_key=True), - sql.Column('option', sql.String(255), primary_key=True), - sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8') - whitelist_table.create(migrate_engine, checkfirst=True) - - sensitive_table = sql.Table( - SENSITIVE_TABLE, - meta, - sql.Column('domain_id', sql.String(64), primary_key=True), - sql.Column('group', sql.String(255), primary_key=True), - sql.Column('option', sql.String(255), primary_key=True), - sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8') - sensitive_table.create(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py deleted file mode 100644 index fe0cee88..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_serialization import jsonutils -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - service_table = sql.Table('service', meta, autoload=True) - services = list(service_table.select().execute()) - - for service in services: - if service.extra is not None: - extra_dict = jsonutils.loads(service.extra) - else: - extra_dict = {} - - # Skip records where service is not null - if extra_dict.get('name') is not None: - continue - # Default the name to empty string - extra_dict['name'] = '' - new_values = { - 'extra': jsonutils.dumps(extra_dict), - } - f = service_table.c.id == service.id - update = service_table.update().where(f).values(new_values) - migrate_engine.execute(update) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py deleted file mode 100644 index b9df1a55..00000000 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/067_drop_redundant_mysql_index.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy - - -def upgrade(migrate_engine): - # NOTE(viktors): Migration 062 removed FK from `assignment` table, but - # MySQL silently creates indexes on FK constraints, so we should remove - # this index manually. - if migrate_engine.name == 'mysql': - meta = sqlalchemy.MetaData(bind=migrate_engine) - table = sqlalchemy.Table('assignment', meta, autoload=True) - for index in table.indexes: - if [c.name for c in index.columns] == ['role_id']: - index.drop(migrate_engine) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/067_kilo.py index 6f326ecf..a6dbed67 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/067_kilo.py @@ -12,18 +12,15 @@ import migrate -from oslo_config import cfg from oslo_log import log import sqlalchemy as sql -from sqlalchemy import orm from keystone.assignment.backends import sql as assignment_sql from keystone.common import sql as ks_sql -from keystone.common.sql import migration_helpers +from keystone.identity.mapping_backends import mapping as mapping_backend LOG = log.getLogger(__name__) -CONF = cfg.CONF def upgrade(migrate_engine): @@ -64,12 +61,12 @@ def upgrade(migrate_engine): sql.Column('id', sql.String(length=64), primary_key=True), sql.Column('legacy_endpoint_id', sql.String(length=64)), sql.Column('interface', sql.String(length=8), nullable=False), - sql.Column('region', sql.String(length=255)), sql.Column('service_id', sql.String(length=64), nullable=False), sql.Column('url', sql.Text, nullable=False), sql.Column('extra', ks_sql.JsonBlob.impl), sql.Column('enabled', sql.Boolean, nullable=False, default=True, server_default='1'), + sql.Column('region_id', sql.String(length=255), nullable=True), mysql_engine='InnoDB', mysql_charset='utf8') @@ -100,6 +97,7 @@ def upgrade(migrate_engine): sql.Column('description', sql.Text), sql.Column('enabled', sql.Boolean), sql.Column('domain_id', sql.String(length=64), nullable=False), + sql.Column('parent_id', sql.String(64), nullable=True), mysql_engine='InnoDB', mysql_charset='utf8') @@ -177,9 +175,9 @@ def upgrade(migrate_engine): region = sql.Table( 'region', meta, - sql.Column('id', sql.String(64), primary_key=True), + sql.Column('id', sql.String(255), primary_key=True), sql.Column('description', sql.String(255), nullable=False), - sql.Column('parent_region_id', sql.String(64), nullable=True), + sql.Column('parent_region_id', sql.String(255), nullable=True), sql.Column('extra', sql.Text()), mysql_engine='InnoDB', mysql_charset='utf8') @@ -202,11 +200,45 @@ def upgrade(migrate_engine): mysql_engine='InnoDB', mysql_charset='utf8') + mapping = sql.Table( + 'id_mapping', + meta, + sql.Column('public_id', sql.String(64), primary_key=True), + sql.Column('domain_id', sql.String(64), nullable=False), + sql.Column('local_id', sql.String(64), nullable=False), + sql.Column('entity_type', sql.Enum( + mapping_backend.EntityType.USER, + mapping_backend.EntityType.GROUP, + name='entity_type'), + nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + + domain_config_whitelist = sql.Table( + 'whitelisted_config', + meta, + sql.Column('domain_id', sql.String(64), primary_key=True), + sql.Column('group', sql.String(255), primary_key=True), + sql.Column('option', sql.String(255), primary_key=True), + sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + + domain_config_sensitive = sql.Table( + 'sensitive_config', + meta, + sql.Column('domain_id', sql.String(64), primary_key=True), + sql.Column('group', sql.String(255), primary_key=True), + sql.Column('option', sql.String(255), primary_key=True), + sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + # create all tables - tables = [credential, domain, endpoint, group, - policy, project, role, service, - token, trust, trust_role, user, - user_group_membership, region, assignment] + tables = [credential, domain, endpoint, group, policy, project, role, + service, token, trust, trust_role, user, user_group_membership, + region, assignment, mapping, domain_config_whitelist, + domain_config_sensitive] for table in tables: try: @@ -229,11 +261,22 @@ def upgrade(migrate_engine): name='ixu_project_name_domain_id').create() migrate.UniqueConstraint(domain.c.name, name='ixu_domain_name').create() + migrate.UniqueConstraint(mapping.c.domain_id, + mapping.c.local_id, + mapping.c.entity_type, + name='domain_id').create() # Indexes sql.Index('ix_token_expires', token.c.expires).create() sql.Index('ix_token_expires_valid', token.c.expires, token.c.valid).create() + sql.Index('ix_actor_id', assignment.c.actor_id).create() + sql.Index('ix_token_user_id', token.c.user_id).create() + sql.Index('ix_token_trust_id', token.c.trust_id).create() + # NOTE(stevemar): The two indexes below were named 'service_id' and + # 'group_id' in 050_fk_consistent_indexes.py, and need to be preserved + sql.Index('service_id', endpoint.c.service_id).create() + sql.Index('group_id', user_group_membership.c.group_id).create() fkeys = [ {'columns': [endpoint.c.service_id], @@ -247,33 +290,28 @@ def upgrade(migrate_engine): 'references':[user.c.id], 'name': 'fk_user_group_membership_user_id'}, - {'columns': [user.c.domain_id], - 'references': [domain.c.id], - 'name': 'fk_user_domain_id'}, - - {'columns': [group.c.domain_id], - 'references': [domain.c.id], - 'name': 'fk_group_domain_id'}, - {'columns': [project.c.domain_id], 'references': [domain.c.id], 'name': 'fk_project_domain_id'}, - {'columns': [assignment.c.role_id], - 'references': [role.c.id]} + {'columns': [endpoint.c.region_id], + 'references': [region.c.id], + 'name': 'fk_endpoint_region_id'}, + + {'columns': [project.c.parent_id], + 'references': [project.c.id], + 'name': 'project_parent_id_fkey'}, ] + if migrate_engine.name == 'sqlite': + # NOTE(stevemar): We need to keep this FK constraint due to 073, but + # only for sqlite, once we collapse 073 we can remove this constraint + fkeys.append( + {'columns': [assignment.c.role_id], + 'references': [role.c.id], + 'name': 'fk_assignment_role_id'}) + for fkey in fkeys: migrate.ForeignKeyConstraint(columns=fkey['columns'], refcolumns=fkey['references'], name=fkey.get('name')).create() - - # Create the default domain. - session = orm.sessionmaker(bind=migrate_engine)() - domain.insert(migration_helpers.get_default_domain()).execute() - session.commit() - - -def downgrade(migrate_engine): - raise NotImplementedError('Downgrade to pre-Icehouse release db schema is ' - 'unsupported.') diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py index ffa210c4..205f809e 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py @@ -18,7 +18,7 @@ from keystone.assignment.backends import sql as assignment_sql def upgrade(migrate_engine): - """Inserts inherited column to assignment table PK contraints. + """Inserts inherited column to assignment table PK constraints. For non-SQLite databases, it changes the constraint in the existing table. @@ -26,7 +26,6 @@ def upgrade(migrate_engine): assignment table with the new PK constraint and migrates the existing data. """ - ASSIGNMENT_TABLE_NAME = 'assignment' metadata = sql.MetaData() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/076_placeholder.py index 8bb40490..9f6e8415 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/076_placeholder.py @@ -10,8 +10,8 @@ # License for the specific language governing permissions and limitations # under the License. -# This is a placeholder for Juno backports. Do not use this number for new -# Kilo work. New Kilo work starts after all the placeholders. +# This is a placeholder for Liberty backports. Do not use this number for new +# Mitaka work. New Mitaka work starts after all the placeholders. def upgrade(migrate_engine): diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/077_placeholder.py index 8bb40490..9f6e8415 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/077_placeholder.py @@ -10,8 +10,8 @@ # License for the specific language governing permissions and limitations # under the License. -# This is a placeholder for Juno backports. Do not use this number for new -# Kilo work. New Kilo work starts after all the placeholders. +# This is a placeholder for Liberty backports. Do not use this number for new +# Mitaka work. New Mitaka work starts after all the placeholders. def upgrade(migrate_engine): diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/078_placeholder.py index 8bb40490..9f6e8415 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/078_placeholder.py @@ -10,8 +10,8 @@ # License for the specific language governing permissions and limitations # under the License. -# This is a placeholder for Juno backports. Do not use this number for new -# Kilo work. New Kilo work starts after all the placeholders. +# This is a placeholder for Liberty backports. Do not use this number for new +# Mitaka work. New Mitaka work starts after all the placeholders. def upgrade(migrate_engine): diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/079_placeholder.py index 8bb40490..9f6e8415 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/079_placeholder.py @@ -10,8 +10,8 @@ # License for the specific language governing permissions and limitations # under the License. -# This is a placeholder for Juno backports. Do not use this number for new -# Kilo work. New Kilo work starts after all the placeholders. +# This is a placeholder for Liberty backports. Do not use this number for new +# Mitaka work. New Mitaka work starts after all the placeholders. def upgrade(migrate_engine): diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/080_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/080_placeholder.py new file mode 100644 index 00000000..9f6e8415 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/080_placeholder.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Liberty backports. Do not use this number for new +# Mitaka work. New Mitaka work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py new file mode 100644 index 00000000..a0c307d0 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py @@ -0,0 +1,54 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.common.sql import migration_helpers + + +def upgrade(migrate_engine): + try: + extension_version = migration_helpers.get_db_version( + extension='endpoint_policy', + engine=migrate_engine) + except Exception: + extension_version = 0 + + # This migration corresponds to endpoint_policy extension migration 1. Only + # update if it has not been run. + if extension_version >= 1: + return + + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata + meta = sql.MetaData() + meta.bind = migrate_engine + + endpoint_policy_table = sql.Table( + 'policy_association', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('policy_id', sql.String(64), + nullable=False), + sql.Column('endpoint_id', sql.String(64), + nullable=True), + sql.Column('service_id', sql.String(64), + nullable=True), + sql.Column('region_id', sql.String(64), + nullable=True), + sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'), + mysql_engine='InnoDB', + mysql_charset='utf8') + + endpoint_policy_table.create(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py new file mode 100644 index 00000000..7e426373 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py @@ -0,0 +1,97 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +import sqlalchemy as sql + +from keystone.common.sql import migration_helpers + +CONF = cfg.CONF +_RELAY_STATE_PREFIX = 'relay_state_prefix' + + +def upgrade(migrate_engine): + try: + extension_version = migration_helpers.get_db_version( + extension='federation', + engine=migrate_engine) + except Exception: + extension_version = 0 + + # This migration corresponds to federation extension migration 8. Only + # update if it has not been run. + if extension_version >= 8: + return + + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata + meta = sql.MetaData() + meta.bind = migrate_engine + + idp_table = sql.Table( + 'identity_provider', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('enabled', sql.Boolean, nullable=False), + sql.Column('description', sql.Text(), nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8') + idp_table.create(migrate_engine, checkfirst=True) + + federation_protocol_table = sql.Table( + 'federation_protocol', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('idp_id', sql.String(64), + sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), + primary_key=True), + sql.Column('mapping_id', sql.String(64), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + federation_protocol_table.create(migrate_engine, checkfirst=True) + + mapping_table = sql.Table( + 'mapping', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('rules', sql.Text(), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + mapping_table.create(migrate_engine, checkfirst=True) + + relay_state_prefix_default = CONF.saml.relay_state_prefix + sp_table = sql.Table( + 'service_provider', + meta, + sql.Column('auth_url', sql.String(256), nullable=False), + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('enabled', sql.Boolean, nullable=False), + sql.Column('description', sql.Text(), nullable=True), + sql.Column('sp_url', sql.String(256), nullable=False), + sql.Column(_RELAY_STATE_PREFIX, sql.String(256), nullable=False, + server_default=relay_state_prefix_default), + mysql_engine='InnoDB', + mysql_charset='utf8') + sp_table.create(migrate_engine, checkfirst=True) + + idp_table = sql.Table('identity_provider', meta, autoload=True) + remote_id_table = sql.Table( + 'idp_remote_ids', + meta, + sql.Column('idp_id', sql.String(64), + sql.ForeignKey('identity_provider.id', ondelete='CASCADE')), + sql.Column('remote_id', sql.String(255), primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8') + remote_id_table.create(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/083_add_oauth1_tables.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/083_add_oauth1_tables.py new file mode 100644 index 00000000..5a859b4b --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/083_add_oauth1_tables.py @@ -0,0 +1,75 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.common.sql import migration_helpers + + +def upgrade(migrate_engine): + try: + extension_version = migration_helpers.get_db_version( + extension='oauth1', + engine=migrate_engine) + except Exception: + extension_version = 0 + + # This migration corresponds to oauth extension migration 5. Only + # update if it has not been run. + if extension_version >= 5: + return + + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata + meta = sql.MetaData() + meta.bind = migrate_engine + + consumer_table = sql.Table( + 'consumer', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('description', sql.String(64), nullable=True), + sql.Column('secret', sql.String(64), nullable=False), + sql.Column('extra', sql.Text(), nullable=False)) + consumer_table.create(migrate_engine, checkfirst=True) + + request_token_table = sql.Table( + 'request_token', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('request_secret', sql.String(64), nullable=False), + sql.Column('verifier', sql.String(64), nullable=True), + sql.Column('authorizing_user_id', sql.String(64), nullable=True), + sql.Column('requested_project_id', sql.String(64), nullable=False), + sql.Column('role_ids', sql.Text(), nullable=True), + sql.Column('consumer_id', sql.String(64), + sql.ForeignKey('consumer.id'), + nullable=False, index=True), + sql.Column('expires_at', sql.String(64), nullable=True)) + request_token_table.create(migrate_engine, checkfirst=True) + + access_token_table = sql.Table( + 'access_token', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('access_secret', sql.String(64), nullable=False), + sql.Column('authorizing_user_id', sql.String(64), + nullable=False, index=True), + sql.Column('project_id', sql.String(64), nullable=False), + sql.Column('role_ids', sql.Text(), nullable=False), + sql.Column('consumer_id', sql.String(64), + sql.ForeignKey('consumer.id'), + nullable=False, index=True), + sql.Column('expires_at', sql.String(64), nullable=True)) + access_token_table.create(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/084_add_revoke_tables.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/084_add_revoke_tables.py new file mode 100644 index 00000000..1a28a53c --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/084_add_revoke_tables.py @@ -0,0 +1,55 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.common.sql import migration_helpers + + +def upgrade(migrate_engine): + try: + extension_version = migration_helpers.get_db_version( + extension='revoke', + engine=migrate_engine) + except Exception: + extension_version = 0 + + # This migration corresponds to revoke extension migration 2. Only + # update if it has not been run. + if extension_version >= 2: + return + + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata + meta = sql.MetaData() + meta.bind = migrate_engine + + service_table = sql.Table( + 'revocation_event', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('domain_id', sql.String(64)), + sql.Column('project_id', sql.String(64)), + sql.Column('user_id', sql.String(64)), + sql.Column('role_id', sql.String(64)), + sql.Column('trust_id', sql.String(64)), + sql.Column('consumer_id', sql.String(64)), + sql.Column('access_token_id', sql.String(64)), + sql.Column('issued_before', sql.DateTime(), nullable=False), + sql.Column('expires_at', sql.DateTime()), + sql.Column('revoked_at', sql.DateTime(), index=True, nullable=False), + sql.Column('audit_id', sql.String(32), nullable=True), + sql.Column('audit_chain_id', sql.String(32), nullable=True)) + + service_table.create(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/085_add_endpoint_filtering_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/085_add_endpoint_filtering_table.py new file mode 100644 index 00000000..5790bd98 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/085_add_endpoint_filtering_table.py @@ -0,0 +1,70 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.common.sql import migration_helpers + + +def upgrade(migrate_engine): + try: + extension_version = migration_helpers.get_db_version( + extension='endpoint_filter', + engine=migrate_engine) + except Exception: + extension_version = 0 + + # This migration corresponds to endpoint_filter extension migration 2. Only + # update if it has not been run. + if extension_version >= 2: + return + + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata + meta = sql.MetaData() + meta.bind = migrate_engine + + EP_GROUP_ID = 'endpoint_group_id' + PROJECT_ID = 'project_id' + + endpoint_filtering_table = sql.Table( + 'project_endpoint', + meta, + sql.Column( + 'endpoint_id', + sql.String(64), + primary_key=True, + nullable=False), + sql.Column( + 'project_id', + sql.String(64), + primary_key=True, + nullable=False)) + endpoint_filtering_table.create(migrate_engine, checkfirst=True) + + endpoint_group_table = sql.Table( + 'endpoint_group', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('name', sql.String(255), nullable=False), + sql.Column('description', sql.Text, nullable=True), + sql.Column('filters', sql.Text(), nullable=False)) + endpoint_group_table.create(migrate_engine, checkfirst=True) + + project_endpoint_group_table = sql.Table( + 'project_endpoint_group', + meta, + sql.Column(EP_GROUP_ID, sql.String(64), + sql.ForeignKey('endpoint_group.id'), nullable=False), + sql.Column(PROJECT_ID, sql.String(64), nullable=False), + sql.PrimaryKeyConstraint(EP_GROUP_ID, PROJECT_ID)) + project_endpoint_group_table.create(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/086_add_duplicate_constraint_trusts.py index 2a98fb90..2b115ea4 100644 --- a/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/086_add_duplicate_constraint_trusts.py @@ -1,3 +1,6 @@ +# Copyright 2015 Intel Corporation +# All Rights Reserved +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -10,12 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. -# This is a placeholder for Icehouse backports. Do not use this number for new -# Juno work. New Juno work starts after all the placeholders. -# -# See blueprint reserved-db-migrations-icehouse and the related discussion: -# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html +from migrate import UniqueConstraint +from sqlalchemy import MetaData, Table def upgrade(migrate_engine): - pass + meta = MetaData(bind=migrate_engine) + trusts = Table('trust', meta, autoload=True) + + UniqueConstraint('trustor_user_id', 'trustee_user_id', 'project_id', + 'impersonation', 'expires_at', table=trusts, + name='duplicate_trust_constraint').create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/087_implied_roles.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/087_implied_roles.py new file mode 100644 index 00000000..7713ce8f --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/087_implied_roles.py @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import migrate +import sqlalchemy as sql + + +ROLE_TABLE = 'role' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + implied_role = sql.Table( + 'implied_role', meta, + sql.Column('prior_role_id', sql.String(length=64), primary_key=True), + sql.Column( + 'implied_role_id', sql.String(length=64), primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8') + implied_role.create() + role = sql.Table(ROLE_TABLE, meta, autoload=True) + fkeys = [ + {'columns': [implied_role.c.prior_role_id], + 'references': [role.c.id]}, + {'columns': [implied_role.c.implied_role_id], + 'references': [role.c.id]}, + ] + for fkey in fkeys: + migrate.ForeignKeyConstraint(columns=fkey['columns'], + refcolumns=fkey['references'], + name=fkey.get('name')).create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/088_domain_specific_roles.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/088_domain_specific_roles.py new file mode 100644 index 00000000..8b792dfa --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/088_domain_specific_roles.py @@ -0,0 +1,60 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy as sql + + +_ROLE_NAME_NEW_CONSTRAINT = 'ixu_role_name_domain_id' +_ROLE_TABLE_NAME = 'role' +_ROLE_NAME_COLUMN_NAME = 'name' +_DOMAIN_ID_COLUMN_NAME = 'domain_id' +_NULL_DOMAIN_ID = '<<null>>' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + role_table = sql.Table(_ROLE_TABLE_NAME, meta, autoload=True) + domain_id = sql.Column(_DOMAIN_ID_COLUMN_NAME, sql.String(64), + nullable=False, server_default=_NULL_DOMAIN_ID) + + # NOTE(morganfainberg): the `role_name` unique constraint is not + # guaranteed to be a fixed name, such as 'ixu_role_name`, so we need to + # search for the correct constraint that only affects role_table.c.name + # and drop that constraint. + to_drop = None + if migrate_engine.name == 'mysql': + for c in role_table.indexes: + if (c.unique and len(c.columns) == 1 and + _ROLE_NAME_COLUMN_NAME in c.columns): + to_drop = c + break + else: + for c in role_table.constraints: + if len(c.columns) == 1 and _ROLE_NAME_COLUMN_NAME in c.columns: + to_drop = c + break + + if to_drop is not None: + migrate.UniqueConstraint(role_table.c.name, + name=to_drop.name).drop() + + # perform changes after constraint is dropped. + if 'domain_id' not in role_table.columns: + # Only create the column if it doesn't already exist. + role_table.create_column(domain_id) + + migrate.UniqueConstraint(role_table.c.name, + role_table.c.domain_id, + name=_ROLE_NAME_NEW_CONSTRAINT).create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/089_add_root_of_all_domains.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/089_add_root_of_all_domains.py new file mode 100644 index 00000000..477c719a --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/089_add_root_of_all_domains.py @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + + +_PROJECT_TABLE_NAME = 'project' +_DOMAIN_TABLE_NAME = 'domain' +NULL_DOMAIN_ID = '<<keystone.domain.root>>' + + +def upgrade(migrate_engine): + + def _generate_root_domain_project(): + # Generate a project that will act as a root for all domains, in order + # for use to be able to use a FK constraint on domain_id. Projects + # acting as a domain will not reference this as their parent_id, just + # as domain_id. + # + # This special project is filtered out by the driver, so is never + # visible to the manager or API. + + project_ref = { + 'id': NULL_DOMAIN_ID, + 'name': NULL_DOMAIN_ID, + 'enabled': False, + 'description': '', + 'domain_id': NULL_DOMAIN_ID, + 'is_domain': True, + 'parent_id': None, + 'extra': '{}' + } + return project_ref + + def _generate_root_domain(): + # Generate a similar root for the domain table, this is an interim + # step so as to allow continuation of current project domain_id FK. + # + # This special domain is filtered out by the driver, so is never + # visible to the manager or API. + + domain_ref = { + 'id': NULL_DOMAIN_ID, + 'name': NULL_DOMAIN_ID, + 'enabled': False, + 'extra': '{}' + } + return domain_ref + + meta = sql.MetaData() + meta.bind = migrate_engine + session = sql.orm.sessionmaker(bind=migrate_engine)() + + project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) + domain_table = sql.Table(_DOMAIN_TABLE_NAME, meta, autoload=True) + + root_domain = _generate_root_domain() + new_entry = domain_table.insert().values(**root_domain) + session.execute(new_entry) + session.commit() + + root_domain_project = _generate_root_domain_project() + new_entry = project_table.insert().values(**root_domain_project) + session.execute(new_entry) + session.commit() + + session.close() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/090_add_local_user_and_password_tables.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/090_add_local_user_and_password_tables.py new file mode 100644 index 00000000..800ba47e --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/090_add_local_user_and_password_tables.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + user = sql.Table('user', meta, autoload=True) + + local_user = sql.Table( + 'local_user', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column('user_id', sql.String(64), + sql.ForeignKey(user.c.id, ondelete='CASCADE'), + nullable=False, unique=True), + sql.Column('domain_id', sql.String(64), nullable=False), + sql.Column('name', sql.String(255), nullable=False), + sql.UniqueConstraint('domain_id', 'name')) + local_user.create(migrate_engine, checkfirst=True) + + password = sql.Table( + 'password', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column('local_user_id', sql.Integer, + sql.ForeignKey(local_user.c.id, ondelete='CASCADE'), + nullable=False), + sql.Column('password', sql.String(128), nullable=False)) + password.create(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/091_migrate_data_to_local_user_and_password_tables.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/091_migrate_data_to_local_user_and_password_tables.py new file mode 100644 index 00000000..1f41fd89 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/091_migrate_data_to_local_user_and_password_tables.py @@ -0,0 +1,66 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy as sql +from sqlalchemy import func + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + user_table = sql.Table('user', meta, autoload=True) + local_user_table = sql.Table('local_user', meta, autoload=True) + password_table = sql.Table('password', meta, autoload=True) + + # migrate data to local_user table + local_user_values = [] + for row in user_table.select().execute(): + # skip the row that already exists in `local_user`, this could + # happen if run into a partially-migrated table due to the + # bug #1549705. + filter_by = local_user_table.c.user_id == row['id'] + user_count = sql.select([func.count()]).select_from( + local_user_table).where(filter_by).execute().fetchone()[0] + if user_count == 0: + local_user_values.append({'user_id': row['id'], + 'domain_id': row['domain_id'], + 'name': row['name']}) + if local_user_values: + local_user_table.insert().values(local_user_values).execute() + + # migrate data to password table + sel = ( + sql.select([user_table, local_user_table], use_labels=True) + .select_from(user_table.join(local_user_table, user_table.c.id == + local_user_table.c.user_id)) + ) + user_rows = sel.execute() + password_values = [] + for row in user_rows: + if row['user_password']: + password_values.append({'local_user_id': row['local_user_id'], + 'password': row['user_password']}) + if password_values: + password_table.insert().values(password_values).execute() + + # remove domain_id and name unique constraint + if migrate_engine.name != 'sqlite': + migrate.UniqueConstraint(user_table.c.domain_id, + user_table.c.name, + name='ixu_user_name_domain_id').drop() + + # drop user columns + user_table.c.domain_id.drop() + user_table.c.name.drop() + user_table.c.password.drop() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/092_make_implied_roles_fks_cascaded.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/092_make_implied_roles_fks_cascaded.py new file mode 100644 index 00000000..5e841899 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/092_make_implied_roles_fks_cascaded.py @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import migrate +import sqlalchemy as sql + + +ROLE_TABLE = 'role' +IMPLIED_ROLE_TABLE = 'implied_role' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + role = sql.Table(ROLE_TABLE, meta, autoload=True) + implied_role = sql.Table(IMPLIED_ROLE_TABLE, meta, autoload=True) + + fkeys = [ + {'columns': [implied_role.c.prior_role_id], + 'references': [role.c.id]}, + {'columns': [implied_role.c.implied_role_id], + 'references': [role.c.id]}, + ] + + # NOTE(stevemar): We need to divide these into two separate loops otherwise + # they may clobber each other and only end up with one foreign key. + for fkey in fkeys: + migrate.ForeignKeyConstraint(columns=fkey['columns'], + refcolumns=fkey['references'], + name=fkey.get('name')).drop() + for fkey in fkeys: + migrate.ForeignKeyConstraint(columns=fkey['columns'], + refcolumns=fkey['references'], + name=fkey.get('name'), + ondelete="CASCADE").create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/093_migrate_domains_to_projects.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/093_migrate_domains_to_projects.py new file mode 100644 index 00000000..f6bba7d9 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/093_migrate_domains_to_projects.py @@ -0,0 +1,125 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import sqlalchemy as sql + +from keystone.common.sql import migration_helpers + + +_PROJECT_TABLE_NAME = 'project' +_DOMAIN_TABLE_NAME = 'domain' +_PARENT_ID_COLUMN_NAME = 'parent_id' +_DOMAIN_ID_COLUMN_NAME = 'domain_id' + +# Above the driver level, the domain_id of a project acting as a domain is +# None. However, in order to enable sql integrity constraints to still operate +# on this column, we create a special "root of all domains" row, with an ID of +# NULL_DOMAIN_ID, which all projects acting as a domain reference in their +# domain_id attribute. This special row, as well as NULL_DOMAIN_ID, are never +# exposed outside of sql driver layer. +NULL_DOMAIN_ID = '<<keystone.domain.root>>' + + +def list_existing_project_constraints(project_table, domain_table): + constraints = [{'table': project_table, + 'fk_column': _PARENT_ID_COLUMN_NAME, + 'ref_column': project_table.c.id}, + {'table': project_table, + 'fk_column': _DOMAIN_ID_COLUMN_NAME, + 'ref_column': domain_table.c.id}] + + return constraints + + +def list_new_project_constraints(project_table): + constraints = [{'table': project_table, + 'fk_column': _PARENT_ID_COLUMN_NAME, + 'ref_column': project_table.c.id}, + {'table': project_table, + 'fk_column': _DOMAIN_ID_COLUMN_NAME, + 'ref_column': project_table.c.id}] + + return constraints + + +def upgrade(migrate_engine): + + def _project_from_domain(domain): + # Creates a project dict with is_domain=True from the provided + # domain. + + description = None + extra = {} + if domain.extra is not None: + # 'description' property is an extra attribute in domains but a + # first class attribute in projects + extra = json.loads(domain.extra) + description = extra.pop('description', None) + + return { + 'id': domain.id, + 'name': domain.name, + 'enabled': domain.enabled, + 'description': description, + 'domain_id': NULL_DOMAIN_ID, + 'is_domain': True, + 'parent_id': None, + 'extra': json.dumps(extra) + } + + meta = sql.MetaData() + meta.bind = migrate_engine + session = sql.orm.sessionmaker(bind=migrate_engine)() + + project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) + domain_table = sql.Table(_DOMAIN_TABLE_NAME, meta, autoload=True) + + # NOTE(htruta): Remove the parent_id constraint during the migration + # because for every root project inside this domain, we will set + # the project domain_id to be its parent_id. We re-enable the constraint + # in the end of this method. We also remove the domain_id constraint, + # while be recreated a FK to the project_id at the end. + migration_helpers.remove_constraints( + list_existing_project_constraints(project_table, domain_table)) + + # For each domain, create a project acting as a domain. We ignore the + # "root of all domains" row, since we already have one of these in the + # project table. + domains = list(domain_table.select().execute()) + for domain in domains: + if domain.id == NULL_DOMAIN_ID: + continue + is_domain_project = _project_from_domain(domain) + new_entry = project_table.insert().values(**is_domain_project) + session.execute(new_entry) + session.commit() + + # For each project, that has no parent (i.e. a top level project), update + # it's parent_id to point at the project acting as its domain. We ignore + # the "root of all domains" row, since its parent_id must always be None. + projects = list(project_table.select().execute()) + for project in projects: + if (project.parent_id is not None or project.is_domain or + project.id == NULL_DOMAIN_ID): + continue + values = {'parent_id': project.domain_id} + update = project_table.update().where( + project_table.c.id == project.id).values(values) + session.execute(update) + session.commit() + + migration_helpers.add_constraints( + list_new_project_constraints(project_table)) + + session.close() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/094_add_federated_user_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/094_add_federated_user_table.py new file mode 100644 index 00000000..6fd3f051 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/094_add_federated_user_table.py @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy as sql + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + user_table = sql.Table('user', meta, autoload=True) + idp_table = sql.Table('identity_provider', meta, autoload=True) + protocol_table = sql.Table('federation_protocol', meta, autoload=True) + + federated_table = sql.Table( + 'federated_user', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column('user_id', sql.String(64), + sql.ForeignKey(user_table.c.id, ondelete='CASCADE'), + nullable=False), + sql.Column('idp_id', sql.String(64), + sql.ForeignKey(idp_table.c.id, ondelete='CASCADE'), + nullable=False), + sql.Column('protocol_id', sql.String(64), nullable=False), + sql.Column('unique_id', sql.String(255), nullable=False), + sql.Column('display_name', sql.String(255), nullable=True), + sql.UniqueConstraint('idp_id', 'protocol_id', 'unique_id')) + federated_table.create(migrate_engine, checkfirst=True) + + migrate.ForeignKeyConstraint( + columns=[federated_table.c.protocol_id, federated_table.c.idp_id], + refcolumns=[protocol_table.c.id, protocol_table.c.idp_id]).create() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/095_add_integer_pkey_to_revocation_event_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/095_add_integer_pkey_to_revocation_event_table.py new file mode 100644 index 00000000..7a75f7b1 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/095_add_integer_pkey_to_revocation_event_table.py @@ -0,0 +1,62 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + # You can specify primary keys when creating tables, however adding + # auto-increment integer primary keys for existing tables is not + # cross-engine compatibility supported. Thus, the approach is to: + # (1) create a new revocation_event table with an int pkey, + # (2) migrate data from the old table to the new table, + # (3) delete the old revocation_event table + # (4) rename the new revocation_event table + revocation_table = sql.Table('revocation_event', meta, autoload=True) + + revocation_table_new = sql.Table( + 'revocation_event_new', + meta, + sql.Column('id', sql.Integer, primary_key=True), + sql.Column('domain_id', sql.String(64)), + sql.Column('project_id', sql.String(64)), + sql.Column('user_id', sql.String(64)), + sql.Column('role_id', sql.String(64)), + sql.Column('trust_id', sql.String(64)), + sql.Column('consumer_id', sql.String(64)), + sql.Column('access_token_id', sql.String(64)), + sql.Column('issued_before', sql.DateTime(), nullable=False), + sql.Column('expires_at', sql.DateTime()), + sql.Column('revoked_at', sql.DateTime(), index=True, nullable=False), + sql.Column('audit_id', sql.String(32), nullable=True), + sql.Column('audit_chain_id', sql.String(32), nullable=True)) + revocation_table_new.create(migrate_engine, checkfirst=True) + + revocation_table_new.insert().from_select(['domain_id', + 'project_id', + 'user_id', + 'role_id', + 'trust_id', + 'consumer_id', + 'access_token_id', + 'issued_before', + 'expires_at', + 'revoked_at', + 'audit_id', + 'audit_chain_id'], + revocation_table.select()) + + revocation_table.drop() + revocation_table_new.rename('revocation_event') diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/096_drop_role_name_constraint.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/096_drop_role_name_constraint.py new file mode 100644 index 00000000..0156de21 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/096_drop_role_name_constraint.py @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy as sql + +_ROLE_TABLE_NAME = 'role' +_ROLE_NAME_COLUMN_NAME = 'name' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + role_table = sql.Table(_ROLE_TABLE_NAME, meta, autoload=True) + + # NOTE(morganfainberg): the `role_name` unique constraint is not + # guaranteed to be named 'ixu_role_name', so we need to search for the + # correct constraint that only affects role_table.c.name and drop + # that constraint. + # + # This is an idempotent change that reflects the fix to migration + # 88 if the role_name unique constraint was not named consistently and + # someone manually fixed the migrations / db without dropping the + # old constraint. + to_drop = None + if migrate_engine.name == 'mysql': + for c in role_table.indexes: + if (c.unique and len(c.columns) == 1 and + _ROLE_NAME_COLUMN_NAME in c.columns): + to_drop = c + break + else: + for c in role_table.constraints: + if len(c.columns) == 1 and _ROLE_NAME_COLUMN_NAME in c.columns: + to_drop = c + break + + if to_drop is not None: + migrate.UniqueConstraint(role_table.c.name, + name=to_drop.name).drop() diff --git a/keystone-moon/keystone/common/sql/migration_helpers.py b/keystone-moon/keystone/common/sql/migration_helpers.py index aaa59f70..40c1fbb5 100644 --- a/keystone-moon/keystone/common/sql/migration_helpers.py +++ b/keystone-moon/keystone/common/sql/migration_helpers.py @@ -21,37 +21,25 @@ import migrate from migrate import exceptions from oslo_config import cfg from oslo_db.sqlalchemy import migration -from oslo_serialization import jsonutils from oslo_utils import importutils import six import sqlalchemy from keystone.common import sql -from keystone.common.sql import migrate_repo from keystone import contrib from keystone import exception from keystone.i18n import _ CONF = cfg.CONF -DEFAULT_EXTENSIONS = ['endpoint_filter', - 'endpoint_policy', - 'federation', - 'oauth1', - 'revoke', - ] - - -def get_default_domain(): - # Return the reference used for the default domain structure during - # sql migrations. - return { - 'id': CONF.identity.default_domain_id, - 'name': 'Default', - 'enabled': True, - 'extra': jsonutils.dumps({'description': 'Owns users and tenants ' - '(i.e. projects) available ' - 'on Identity API v2.'})} +DEFAULT_EXTENSIONS = [] + +MIGRATED_EXTENSIONS = ['endpoint_policy', + 'federation', + 'oauth1', + 'revoke', + 'endpoint_filter' + ] # Different RDBMSs use different schemes for naming the Foreign Key @@ -117,9 +105,8 @@ def rename_tables_with_constraints(renames, constraints, engine): `renames` is a dict, mapping {'to_table_name': from_table, ...} """ - if engine.name != 'sqlite': - # Sqlite doesn't support constraints, so nothing to remove. + # SQLite doesn't support constraints, so nothing to remove. remove_constraints(constraints) for to_table_name in renames: @@ -141,11 +128,34 @@ def find_migrate_repo(package=None, repo_name='migrate_repo'): def _sync_common_repo(version): abs_path = find_migrate_repo() - init_version = migrate_repo.DB_INIT_VERSION - engine = sql.get_engine() - _assert_not_schema_downgrade(version=version) - migration.db_sync(engine, abs_path, version=version, - init_version=init_version, sanity_check=False) + init_version = get_init_version() + with sql.session_for_write() as session: + engine = session.get_bind() + _assert_not_schema_downgrade(version=version) + migration.db_sync(engine, abs_path, version=version, + init_version=init_version, sanity_check=False) + + +def get_init_version(abs_path=None): + """Get the initial version of a migrate repository + + :param abs_path: Absolute path to migrate repository. + :return: initial version number or None, if DB is empty. + """ + if abs_path is None: + abs_path = find_migrate_repo() + + repo = migrate.versioning.repository.Repository(abs_path) + + # Sadly, Repository has a `latest` but not an `oldest`. + # The value is a VerNum object which needs to be converted into an int. + oldest = int(min(repo.versions.versions)) + + if oldest < 1: + return None + + # The initial version is one less + return oldest - 1 def _assert_not_schema_downgrade(extension=None, version=None): @@ -153,40 +163,46 @@ def _assert_not_schema_downgrade(extension=None, version=None): try: current_ver = int(six.text_type(get_db_version(extension))) if int(version) < current_ver: - raise migration.exception.DbMigrationError() - except exceptions.DatabaseNotControlledError: + raise migration.exception.DbMigrationError( + _("Unable to downgrade schema")) + except exceptions.DatabaseNotControlledError: # nosec # NOTE(morganfainberg): The database is not controlled, this action # cannot be a downgrade. pass def _sync_extension_repo(extension, version): - init_version = 0 - engine = sql.get_engine() + if extension in MIGRATED_EXTENSIONS: + raise exception.MigrationMovedFailure(extension=extension) + + with sql.session_for_write() as session: + engine = session.get_bind() - try: - package_name = '.'.join((contrib.__name__, extension)) - package = importutils.import_module(package_name) - except ImportError: - raise ImportError(_("%s extension does not exist.") - % package_name) - try: - abs_path = find_migrate_repo(package) try: - migration.db_version_control(sql.get_engine(), abs_path) - # Register the repo with the version control API - # If it already knows about the repo, it will throw - # an exception that we can safely ignore - except exceptions.DatabaseAlreadyControlledError: - pass - except exception.MigrationNotProvided as e: - print(e) - sys.exit(1) + package_name = '.'.join((contrib.__name__, extension)) + package = importutils.import_module(package_name) + except ImportError: + raise ImportError(_("%s extension does not exist.") + % package_name) + try: + abs_path = find_migrate_repo(package) + try: + migration.db_version_control(engine, abs_path) + # Register the repo with the version control API + # If it already knows about the repo, it will throw + # an exception that we can safely ignore + except exceptions.DatabaseAlreadyControlledError: # nosec + pass + except exception.MigrationNotProvided as e: + print(e) + sys.exit(1) + + _assert_not_schema_downgrade(extension=extension, version=version) - _assert_not_schema_downgrade(extension=extension, version=version) + init_version = get_init_version(abs_path=abs_path) - migration.db_sync(engine, abs_path, version=version, - init_version=init_version, sanity_check=False) + migration.db_sync(engine, abs_path, version=version, + init_version=init_version, sanity_check=False) def sync_database_to_version(extension=None, version=None): @@ -203,8 +219,10 @@ def sync_database_to_version(extension=None, version=None): def get_db_version(extension=None): if not extension: - return migration.db_version(sql.get_engine(), find_migrate_repo(), - migrate_repo.DB_INIT_VERSION) + with sql.session_for_write() as session: + return migration.db_version(session.get_bind(), + find_migrate_repo(), + get_init_version()) try: package_name = '.'.join((contrib.__name__, extension)) @@ -213,8 +231,9 @@ def get_db_version(extension=None): raise ImportError(_("%s extension does not exist.") % package_name) - return migration.db_version( - sql.get_engine(), find_migrate_repo(package), 0) + with sql.session_for_write() as session: + return migration.db_version( + session.get_bind(), find_migrate_repo(package), 0) def print_db_version(extension=None): |