diff options
Diffstat (limited to 'keystone-moon/keystone/common/sql')
31 files changed, 1897 insertions, 0 deletions
diff --git a/keystone-moon/keystone/common/sql/__init__.py b/keystone-moon/keystone/common/sql/__init__.py new file mode 100644 index 00000000..84e0fb83 --- /dev/null +++ b/keystone-moon/keystone/common/sql/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystone.common.sql.core import * # noqa diff --git a/keystone-moon/keystone/common/sql/core.py b/keystone-moon/keystone/common/sql/core.py new file mode 100644 index 00000000..bf168701 --- /dev/null +++ b/keystone-moon/keystone/common/sql/core.py @@ -0,0 +1,431 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""SQL backends for the various services. + +Before using this module, call initialize(). This has to be done before +CONF() because it sets up configuration options. + +""" +import contextlib +import functools + +from oslo_config import cfg +from oslo_db import exception as db_exception +from oslo_db import options as db_options +from oslo_db.sqlalchemy import models +from oslo_db.sqlalchemy import session as db_session +from oslo_log import log +from oslo_serialization import jsonutils +import six +import sqlalchemy as sql +from sqlalchemy.ext import declarative +from sqlalchemy.orm.attributes import flag_modified, InstrumentedAttribute +from sqlalchemy import types as sql_types + +from keystone.common import utils +from keystone import exception +from keystone.i18n import _ + + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + +ModelBase = declarative.declarative_base() + + +# For exporting to other modules +Column = sql.Column +Index = sql.Index +String = sql.String +Integer = sql.Integer +Enum = sql.Enum +ForeignKey = sql.ForeignKey +DateTime = sql.DateTime +IntegrityError = sql.exc.IntegrityError +DBDuplicateEntry = db_exception.DBDuplicateEntry +OperationalError = sql.exc.OperationalError +NotFound = sql.orm.exc.NoResultFound +Boolean = sql.Boolean +Text = sql.Text +UniqueConstraint = sql.UniqueConstraint +PrimaryKeyConstraint = sql.PrimaryKeyConstraint +joinedload = sql.orm.joinedload +# Suppress flake8's unused import warning for flag_modified: +flag_modified = flag_modified + + +def initialize(): + """Initialize the module.""" + + db_options.set_defaults( + CONF, + connection="sqlite:///keystone.db") + + +def initialize_decorator(init): + """Ensure that the length of string field do not exceed the limit. + + This decorator check the initialize arguments, to make sure the + length of string field do not exceed the length limit, or raise a + 'StringLengthExceeded' exception. + + Use decorator instead of inheritance, because the metaclass will + check the __tablename__, primary key columns, etc. at the class + definition. + + """ + def initialize(self, *args, **kwargs): + cls = type(self) + for k, v in kwargs.items(): + if hasattr(cls, k): + attr = getattr(cls, k) + if isinstance(attr, InstrumentedAttribute): + column = attr.property.columns[0] + if isinstance(column.type, String): + if not isinstance(v, six.text_type): + v = six.text_type(v) + if column.type.length and column.type.length < len(v): + raise exception.StringLengthExceeded( + string=v, type=k, length=column.type.length) + + init(self, *args, **kwargs) + return initialize + +ModelBase.__init__ = initialize_decorator(ModelBase.__init__) + + +# Special Fields +class JsonBlob(sql_types.TypeDecorator): + + impl = sql.Text + + def process_bind_param(self, value, dialect): + return jsonutils.dumps(value) + + def process_result_value(self, value, dialect): + return jsonutils.loads(value) + + +class DictBase(models.ModelBase): + attributes = [] + + @classmethod + def from_dict(cls, d): + new_d = d.copy() + + new_d['extra'] = {k: new_d.pop(k) for k in six.iterkeys(d) + if k not in cls.attributes and k != 'extra'} + + return cls(**new_d) + + def to_dict(self, include_extra_dict=False): + """Returns the model's attributes as a dictionary. + + If include_extra_dict is True, 'extra' attributes are literally + included in the resulting dictionary twice, for backwards-compatibility + with a broken implementation. + + """ + d = self.extra.copy() + for attr in self.__class__.attributes: + d[attr] = getattr(self, attr) + + if include_extra_dict: + d['extra'] = self.extra.copy() + + return d + + def __getitem__(self, key): + if key in self.extra: + return self.extra[key] + return getattr(self, key) + + +class ModelDictMixin(object): + + @classmethod + def from_dict(cls, d): + """Returns a model instance from a dictionary.""" + return cls(**d) + + def to_dict(self): + """Returns the model's attributes as a dictionary.""" + names = (column.name for column in self.__table__.columns) + return {name: getattr(self, name) for name in names} + + +_engine_facade = None + + +def _get_engine_facade(): + global _engine_facade + + if not _engine_facade: + _engine_facade = db_session.EngineFacade.from_config(CONF) + + return _engine_facade + + +def cleanup(): + global _engine_facade + + _engine_facade = None + + +def get_engine(): + return _get_engine_facade().get_engine() + + +def get_session(expire_on_commit=False): + return _get_engine_facade().get_session(expire_on_commit=expire_on_commit) + + +@contextlib.contextmanager +def transaction(expire_on_commit=False): + """Return a SQLAlchemy session in a scoped transaction.""" + session = get_session(expire_on_commit=expire_on_commit) + with session.begin(): + yield session + + +def truncated(f): + """Ensure list truncation is detected in Driver list entity methods. + + This is designed to wrap and sql Driver list_{entity} methods in order to + calculate if the resultant list has been truncated. Provided a limit dict + is found in the hints list, we increment the limit by one so as to ask the + wrapped function for one more entity than the limit, and then once the list + has been generated, we check to see if the original limit has been + exceeded, in which case we truncate back to that limit and set the + 'truncated' boolean to 'true' in the hints limit dict. + + """ + @functools.wraps(f) + def wrapper(self, hints, *args, **kwargs): + if not hasattr(hints, 'limit'): + raise exception.UnexpectedError( + _('Cannot truncate a driver call without hints list as ' + 'first parameter after self ')) + + if hints.limit is None: + return f(self, hints, *args, **kwargs) + + # A limit is set, so ask for one more entry than we need + list_limit = hints.limit['limit'] + hints.set_limit(list_limit + 1) + ref_list = f(self, hints, *args, **kwargs) + + # If we got more than the original limit then trim back the list and + # mark it truncated. In both cases, make sure we set the limit back + # to its original value. + if len(ref_list) > list_limit: + hints.set_limit(list_limit, truncated=True) + return ref_list[:list_limit] + else: + hints.set_limit(list_limit) + return ref_list + return wrapper + + +def _filter(model, query, hints): + """Applies filtering to a query. + + :param model: the table model in question + :param query: query to apply filters to + :param hints: contains the list of filters yet to be satisfied. + Any filters satisfied here will be removed so that + the caller will know if any filters remain. + + :returns query: query, updated with any filters satisfied + + """ + def inexact_filter(model, query, filter_, satisfied_filters, hints): + """Applies an inexact filter to a query. + + :param model: the table model in question + :param query: query to apply filters to + :param filter_: the dict that describes this filter + :param satisfied_filters: a cumulative list of satisfied filters, to + which filter_ will be added if it is + satisfied. + :param hints: contains the list of filters yet to be satisfied. + + :returns query: query updated to add any inexact filters we could + satisfy + + """ + column_attr = getattr(model, filter_['name']) + + # TODO(henry-nash): Sqlalchemy 0.7 defaults to case insensitivity + # so once we find a way of changing that (maybe on a call-by-call + # basis), we can add support for the case sensitive versions of + # the filters below. For now, these case sensitive versions will + # be handled at the controller level. + + if filter_['case_sensitive']: + return query + + if filter_['comparator'] == 'contains': + query_term = column_attr.ilike('%%%s%%' % filter_['value']) + elif filter_['comparator'] == 'startswith': + query_term = column_attr.ilike('%s%%' % filter_['value']) + elif filter_['comparator'] == 'endswith': + query_term = column_attr.ilike('%%%s' % filter_['value']) + else: + # It's a filter we don't understand, so let the caller + # work out if they need to do something with it. + return query + + satisfied_filters.append(filter_) + return query.filter(query_term) + + def exact_filter( + model, filter_, satisfied_filters, cumulative_filter_dict, hints): + """Applies an exact filter to a query. + + :param model: the table model in question + :param filter_: the dict that describes this filter + :param satisfied_filters: a cumulative list of satisfied filters, to + which filter_ will be added if it is + satisfied. + :param cumulative_filter_dict: a dict that describes the set of + exact filters built up so far + :param hints: contains the list of filters yet to be satisfied. + + :returns: updated cumulative dict + + """ + key = filter_['name'] + if isinstance(getattr(model, key).property.columns[0].type, + sql.types.Boolean): + cumulative_filter_dict[key] = ( + utils.attr_as_boolean(filter_['value'])) + else: + cumulative_filter_dict[key] = filter_['value'] + satisfied_filters.append(filter_) + return cumulative_filter_dict + + filter_dict = {} + satisfied_filters = [] + for filter_ in hints.filters: + if filter_['name'] not in model.attributes: + continue + if filter_['comparator'] == 'equals': + filter_dict = exact_filter( + model, filter_, satisfied_filters, filter_dict, hints) + else: + query = inexact_filter( + model, query, filter_, satisfied_filters, hints) + + # Apply any exact filters we built up + if filter_dict: + query = query.filter_by(**filter_dict) + + # Remove satisfied filters, then the caller will know remaining filters + for filter_ in satisfied_filters: + hints.filters.remove(filter_) + + return query + + +def _limit(query, hints): + """Applies a limit to a query. + + :param query: query to apply filters to + :param hints: contains the list of filters and limit details. + + :returns updated query + + """ + # NOTE(henry-nash): If we were to implement pagination, then we + # we would expand this method to support pagination and limiting. + + # If we satisfied all the filters, set an upper limit if supplied + if hints.limit: + query = query.limit(hints.limit['limit']) + return query + + +def filter_limit_query(model, query, hints): + """Applies filtering and limit to a query. + + :param model: table model + :param query: query to apply filters to + :param hints: contains the list of filters and limit details. This may + be None, indicating that there are no filters or limits + to be applied. If it's not None, then any filters + satisfied here will be removed so that the caller will + know if any filters remain. + + :returns: updated query + + """ + if hints is None: + return query + + # First try and satisfy any filters + query = _filter(model, query, hints) + + # NOTE(henry-nash): Any unsatisfied filters will have been left in + # the hints list for the controller to handle. We can only try and + # limit here if all the filters are already satisfied since, if not, + # doing so might mess up the final results. If there are still + # unsatisfied filters, we have to leave any limiting to the controller + # as well. + + if not hints.filters: + return _limit(query, hints) + else: + return query + + +def handle_conflicts(conflict_type='object'): + """Converts select sqlalchemy exceptions into HTTP 409 Conflict.""" + _conflict_msg = 'Conflict %(conflict_type)s: %(details)s' + + def decorator(method): + @functools.wraps(method) + def wrapper(*args, **kwargs): + try: + return method(*args, **kwargs) + except db_exception.DBDuplicateEntry as e: + # LOG the exception for debug purposes, do not send the + # exception details out with the raised Conflict exception + # as it can contain raw SQL. + LOG.debug(_conflict_msg, {'conflict_type': conflict_type, + 'details': six.text_type(e)}) + raise exception.Conflict(type=conflict_type, + details=_('Duplicate Entry')) + except db_exception.DBError as e: + # TODO(blk-u): inspecting inner_exception breaks encapsulation; + # oslo_db should provide exception we need. + if isinstance(e.inner_exception, IntegrityError): + # LOG the exception for debug purposes, do not send the + # exception details out with the raised Conflict exception + # as it can contain raw SQL. + LOG.debug(_conflict_msg, {'conflict_type': conflict_type, + 'details': six.text_type(e)}) + # NOTE(morganfainberg): This is really a case where the SQL + # failed to store the data. This is not something that the + # user has done wrong. Example would be a ForeignKey is + # missing; the code that is executed before reaching the + # SQL writing to the DB should catch the issue. + raise exception.UnexpectedError( + _('An unexpected error occurred when trying to ' + 'store %s') % conflict_type) + raise + + return wrapper + return decorator diff --git a/keystone-moon/keystone/common/sql/migrate_repo/README b/keystone-moon/keystone/common/sql/migrate_repo/README new file mode 100644 index 00000000..6218f8ca --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/keystone-moon/keystone/common/sql/migrate_repo/__init__.py b/keystone-moon/keystone/common/sql/migrate_repo/__init__.py new file mode 100644 index 00000000..f73dfc12 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2014 Mirantis.inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +DB_INIT_VERSION = 43 diff --git a/keystone-moon/keystone/common/sql/migrate_repo/manage.py b/keystone-moon/keystone/common/sql/migrate_repo/manage.py new file mode 100644 index 00000000..39fa3892 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/manage.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main + +if __name__ == '__main__': + main(debug='False') diff --git a/keystone-moon/keystone/common/sql/migrate_repo/migrate.cfg b/keystone-moon/keystone/common/sql/migrate_repo/migrate.cfg new file mode 100644 index 00000000..db531bb4 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/migrate.cfg @@ -0,0 +1,25 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=keystone + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering=False diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py new file mode 100644 index 00000000..6f326ecf --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py @@ -0,0 +1,279 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import migrate +from oslo_config import cfg +from oslo_log import log +import sqlalchemy as sql +from sqlalchemy import orm + +from keystone.assignment.backends import sql as assignment_sql +from keystone.common import sql as ks_sql +from keystone.common.sql import migration_helpers + + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + if migrate_engine.name == 'mysql': + # In Folsom we explicitly converted migrate_version to UTF8. + migrate_engine.execute( + 'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8') + # Set default DB charset to UTF8. + migrate_engine.execute( + 'ALTER DATABASE %s DEFAULT CHARACTER SET utf8' % + migrate_engine.url.database) + + credential = sql.Table( + 'credential', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('user_id', sql.String(length=64), nullable=False), + sql.Column('project_id', sql.String(length=64)), + sql.Column('blob', ks_sql.JsonBlob, nullable=False), + sql.Column('type', sql.String(length=255), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + domain = sql.Table( + 'domain', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=64), nullable=False), + sql.Column('enabled', sql.Boolean, default=True, nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + endpoint = sql.Table( + 'endpoint', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('legacy_endpoint_id', sql.String(length=64)), + sql.Column('interface', sql.String(length=8), nullable=False), + sql.Column('region', sql.String(length=255)), + sql.Column('service_id', sql.String(length=64), nullable=False), + sql.Column('url', sql.Text, nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('enabled', sql.Boolean, nullable=False, default=True, + server_default='1'), + mysql_engine='InnoDB', + mysql_charset='utf8') + + group = sql.Table( + 'group', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('domain_id', sql.String(length=64), nullable=False), + sql.Column('name', sql.String(length=64), nullable=False), + sql.Column('description', sql.Text), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + policy = sql.Table( + 'policy', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('type', sql.String(length=255), nullable=False), + sql.Column('blob', ks_sql.JsonBlob, nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + project = sql.Table( + 'project', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=64), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('description', sql.Text), + sql.Column('enabled', sql.Boolean), + sql.Column('domain_id', sql.String(length=64), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + + role = sql.Table( + 'role', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=255), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + service = sql.Table( + 'service', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('type', sql.String(length=255)), + sql.Column('enabled', sql.Boolean, nullable=False, default=True, + server_default='1'), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + token = sql.Table( + 'token', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('expires', sql.DateTime, default=None), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('valid', sql.Boolean, default=True, nullable=False), + sql.Column('trust_id', sql.String(length=64)), + sql.Column('user_id', sql.String(length=64)), + mysql_engine='InnoDB', + mysql_charset='utf8') + + trust = sql.Table( + 'trust', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('trustor_user_id', sql.String(length=64), nullable=False), + sql.Column('trustee_user_id', sql.String(length=64), nullable=False), + sql.Column('project_id', sql.String(length=64)), + sql.Column('impersonation', sql.Boolean, nullable=False), + sql.Column('deleted_at', sql.DateTime), + sql.Column('expires_at', sql.DateTime), + sql.Column('remaining_uses', sql.Integer, nullable=True), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + trust_role = sql.Table( + 'trust_role', meta, + sql.Column('trust_id', sql.String(length=64), primary_key=True, + nullable=False), + sql.Column('role_id', sql.String(length=64), primary_key=True, + nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + + user = sql.Table( + 'user', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=255), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('password', sql.String(length=128)), + sql.Column('enabled', sql.Boolean), + sql.Column('domain_id', sql.String(length=64), nullable=False), + sql.Column('default_project_id', sql.String(length=64)), + mysql_engine='InnoDB', + mysql_charset='utf8') + + user_group_membership = sql.Table( + 'user_group_membership', meta, + sql.Column('user_id', sql.String(length=64), primary_key=True), + sql.Column('group_id', sql.String(length=64), primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8') + + region = sql.Table( + 'region', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('description', sql.String(255), nullable=False), + sql.Column('parent_region_id', sql.String(64), nullable=True), + sql.Column('extra', sql.Text()), + mysql_engine='InnoDB', + mysql_charset='utf8') + + assignment = sql.Table( + 'assignment', + meta, + sql.Column('type', sql.Enum( + assignment_sql.AssignmentType.USER_PROJECT, + assignment_sql.AssignmentType.GROUP_PROJECT, + assignment_sql.AssignmentType.USER_DOMAIN, + assignment_sql.AssignmentType.GROUP_DOMAIN, + name='type'), + nullable=False), + sql.Column('actor_id', sql.String(64), nullable=False), + sql.Column('target_id', sql.String(64), nullable=False), + sql.Column('role_id', sql.String(64), nullable=False), + sql.Column('inherited', sql.Boolean, default=False, nullable=False), + sql.PrimaryKeyConstraint('type', 'actor_id', 'target_id', 'role_id'), + mysql_engine='InnoDB', + mysql_charset='utf8') + + # create all tables + tables = [credential, domain, endpoint, group, + policy, project, role, service, + token, trust, trust_role, user, + user_group_membership, region, assignment] + + for table in tables: + try: + table.create() + except Exception: + LOG.exception('Exception while creating table: %r', table) + raise + + # Unique Constraints + migrate.UniqueConstraint(user.c.domain_id, + user.c.name, + name='ixu_user_name_domain_id').create() + migrate.UniqueConstraint(group.c.domain_id, + group.c.name, + name='ixu_group_name_domain_id').create() + migrate.UniqueConstraint(role.c.name, + name='ixu_role_name').create() + migrate.UniqueConstraint(project.c.domain_id, + project.c.name, + name='ixu_project_name_domain_id').create() + migrate.UniqueConstraint(domain.c.name, + name='ixu_domain_name').create() + + # Indexes + sql.Index('ix_token_expires', token.c.expires).create() + sql.Index('ix_token_expires_valid', token.c.expires, + token.c.valid).create() + + fkeys = [ + {'columns': [endpoint.c.service_id], + 'references': [service.c.id]}, + + {'columns': [user_group_membership.c.group_id], + 'references': [group.c.id], + 'name': 'fk_user_group_membership_group_id'}, + + {'columns': [user_group_membership.c.user_id], + 'references':[user.c.id], + 'name': 'fk_user_group_membership_user_id'}, + + {'columns': [user.c.domain_id], + 'references': [domain.c.id], + 'name': 'fk_user_domain_id'}, + + {'columns': [group.c.domain_id], + 'references': [domain.c.id], + 'name': 'fk_group_domain_id'}, + + {'columns': [project.c.domain_id], + 'references': [domain.c.id], + 'name': 'fk_project_domain_id'}, + + {'columns': [assignment.c.role_id], + 'references': [role.c.id]} + ] + + for fkey in fkeys: + migrate.ForeignKeyConstraint(columns=fkey['columns'], + refcolumns=fkey['references'], + name=fkey.get('name')).create() + + # Create the default domain. + session = orm.sessionmaker(bind=migrate_engine)() + domain.insert(migration_helpers.get_default_domain()).execute() + session.commit() + + +def downgrade(migrate_engine): + raise NotImplementedError('Downgrade to pre-Icehouse release db schema is ' + 'unsupported.') diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py new file mode 100644 index 00000000..b6f40719 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Icehouse backports. Do not use this number for new +# Juno work. New Juno work starts after all the placeholders. +# +# See blueprint reserved-db-migrations-icehouse and the related discussion: +# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py new file mode 100644 index 00000000..b6f40719 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Icehouse backports. Do not use this number for new +# Juno work. New Juno work starts after all the placeholders. +# +# See blueprint reserved-db-migrations-icehouse and the related discussion: +# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py new file mode 100644 index 00000000..b6f40719 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Icehouse backports. Do not use this number for new +# Juno work. New Juno work starts after all the placeholders. +# +# See blueprint reserved-db-migrations-icehouse and the related discussion: +# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py new file mode 100644 index 00000000..b6f40719 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Icehouse backports. Do not use this number for new +# Juno work. New Juno work starts after all the placeholders. +# +# See blueprint reserved-db-migrations-icehouse and the related discussion: +# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py new file mode 100644 index 00000000..b6f40719 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Icehouse backports. Do not use this number for new +# Juno work. New Juno work starts after all the placeholders. +# +# See blueprint reserved-db-migrations-icehouse and the related discussion: +# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py new file mode 100644 index 00000000..535a0944 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py @@ -0,0 +1,49 @@ +# Copyright 2014 Mirantis.inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sa + + +def upgrade(migrate_engine): + + if migrate_engine.name == 'mysql': + meta = sa.MetaData(bind=migrate_engine) + endpoint = sa.Table('endpoint', meta, autoload=True) + + # NOTE(i159): MySQL requires indexes on referencing columns, and those + # indexes create automatically. That those indexes will have different + # names, depending on version of MySQL used. We shoud make this naming + # consistent, by reverting index name to a consistent condition. + if any(i for i in endpoint.indexes if + i.columns.keys() == ['service_id'] and i.name != 'service_id'): + # NOTE(i159): by this action will be made re-creation of an index + # with the new name. This can be considered as renaming under the + # MySQL rules. + sa.Index('service_id', endpoint.c.service_id).create() + + user_group_membership = sa.Table('user_group_membership', + meta, autoload=True) + + if any(i for i in user_group_membership.indexes if + i.columns.keys() == ['group_id'] and i.name != 'group_id'): + sa.Index('group_id', user_group_membership.c.group_id).create() + + +def downgrade(migrate_engine): + # NOTE(i159): index exists only in MySQL schemas, and got an inconsistent + # name only when MySQL 5.5 renamed it after re-creation + # (during migrations). So we just fixed inconsistency, there is no + # necessity to revert it. + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py new file mode 100644 index 00000000..074fbb63 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py @@ -0,0 +1,49 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.identity.mapping_backends import mapping + + +MAPPING_TABLE = 'id_mapping' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + mapping_table = sql.Table( + MAPPING_TABLE, + meta, + sql.Column('public_id', sql.String(64), primary_key=True), + sql.Column('domain_id', sql.String(64), nullable=False), + sql.Column('local_id', sql.String(64), nullable=False), + sql.Column('entity_type', sql.Enum( + mapping.EntityType.USER, + mapping.EntityType.GROUP, + name='entity_type'), + nullable=False), + sql.UniqueConstraint('domain_id', 'local_id', 'entity_type'), + mysql_engine='InnoDB', + mysql_charset='utf8') + mapping_table.create(migrate_engine, checkfirst=True) + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + assignment = sql.Table(MAPPING_TABLE, meta, autoload=True) + assignment.drop(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py new file mode 100644 index 00000000..9f1fd9f0 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py @@ -0,0 +1,34 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +_REGION_TABLE_NAME = 'region' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) + url_column = sql.Column('url', sql.String(255), nullable=True) + region_table.create_column(url_column) + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) + region_table.drop_column('url') diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py new file mode 100644 index 00000000..6dc0004f --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py @@ -0,0 +1,156 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +"""Migrated the endpoint 'region' column to 'region_id. + +In addition to the rename, the new column is made a foreign key to the +respective 'region' in the region table, ensuring that we auto-create +any regions that are missing. Further, since the old region column +was 255 chars, and the id column in the region table is 64 chars, the size +of the id column in the region table is increased to match. + +To Upgrade: + + +Region Table + +Increase the size of the if column in the region table + +Endpoint Table + +a. Add the endpoint region_id column, that is a foreign key to the region table +b. For each endpoint + i. Ensure there is matching region in region table, and if not, create it + ii. Assign the id to the region_id column +c. Remove the column region + + +To Downgrade: + +Endpoint Table + +a. Add back in the region column +b. For each endpoint + i. Copy the region_id column to the region column +c. Remove the column region_id + +Region Table + +Decrease the size of the id column in the region table, making sure that +we don't get classing primary keys. + +""" + +import migrate +import six +import sqlalchemy as sql +from sqlalchemy.orm import sessionmaker + + +def _migrate_to_region_id(migrate_engine, region_table, endpoint_table): + endpoints = list(endpoint_table.select().execute()) + + for endpoint in endpoints: + if endpoint.region is None: + continue + + region = list(region_table.select( + whereclause=region_table.c.id == endpoint.region).execute()) + if len(region) == 1: + region_id = region[0].id + else: + region_id = endpoint.region + region = {'id': region_id, + 'description': '', + 'extra': '{}'} + session = sessionmaker(bind=migrate_engine)() + region_table.insert(region).execute() + session.commit() + + new_values = {'region_id': region_id} + f = endpoint_table.c.id == endpoint.id + update = endpoint_table.update().where(f).values(new_values) + migrate_engine.execute(update) + + migrate.ForeignKeyConstraint( + columns=[endpoint_table.c.region_id], + refcolumns=[region_table.c.id], + name='fk_endpoint_region_id').create() + + +def _migrate_to_region(migrate_engine, region_table, endpoint_table): + endpoints = list(endpoint_table.select().execute()) + + for endpoint in endpoints: + new_values = {'region': endpoint.region_id} + f = endpoint_table.c.id == endpoint.id + update = endpoint_table.update().where(f).values(new_values) + migrate_engine.execute(update) + + if 'sqlite' != migrate_engine.name: + migrate.ForeignKeyConstraint( + columns=[endpoint_table.c.region_id], + refcolumns=[region_table.c.id], + name='fk_endpoint_region_id').drop() + endpoint_table.c.region_id.drop() + + +def _prepare_regions_for_id_truncation(migrate_engine, region_table): + """Ensure there are no IDs that are bigger than 64 chars. + + The size of the id and parent_id fields where increased from 64 to 255 + during the upgrade. On downgrade we have to make sure that the ids can + fit in the new column size. For rows with ids greater than this, we have + no choice but to dump them. + + """ + for region in list(region_table.select().execute()): + if (len(six.text_type(region.id)) > 64 or + len(six.text_type(region.parent_region_id)) > 64): + delete = region_table.delete(region_table.c.id == region.id) + migrate_engine.execute(delete) + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + region_table = sql.Table('region', meta, autoload=True) + region_table.c.id.alter(type=sql.String(length=255)) + region_table.c.parent_region_id.alter(type=sql.String(length=255)) + endpoint_table = sql.Table('endpoint', meta, autoload=True) + region_id_column = sql.Column('region_id', + sql.String(length=255), nullable=True) + region_id_column.create(endpoint_table) + + _migrate_to_region_id(migrate_engine, region_table, endpoint_table) + + endpoint_table.c.region.drop() + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + region_table = sql.Table('region', meta, autoload=True) + endpoint_table = sql.Table('endpoint', meta, autoload=True) + region_column = sql.Column('region', sql.String(length=255)) + region_column.create(endpoint_table) + + _migrate_to_region(migrate_engine, region_table, endpoint_table) + _prepare_regions_for_id_truncation(migrate_engine, region_table) + + region_table.c.id.alter(type=sql.String(length=64)) + region_table.c.parent_region_id.alter(type=sql.String(length=64)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py new file mode 100644 index 00000000..33b13b7d --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py @@ -0,0 +1,35 @@ +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +ASSIGNMENT_TABLE = 'assignment' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True) + idx = sql.Index('ix_actor_id', assignment.c.actor_id) + idx.create(migrate_engine) + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True) + idx = sql.Index('ix_actor_id', assignment.c.actor_id) + idx.drop(migrate_engine) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py new file mode 100644 index 00000000..1cfddd3f --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Add indexes to `user_id` and `trust_id` columns for the `token` table.""" + +import sqlalchemy as sql + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + token = sql.Table('token', meta, autoload=True) + + sql.Index('ix_token_user_id', token.c.user_id).create() + sql.Index('ix_token_trust_id', token.c.trust_id).create() + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + token = sql.Table('token', meta, autoload=True) + + sql.Index('ix_token_user_id', token.c.user_id).drop() + sql.Index('ix_token_trust_id', token.c.trust_id).drop() diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py new file mode 100644 index 00000000..5f82254f --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Juno backports. Do not use this number for new +# Kilo work. New Kilo work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py new file mode 100644 index 00000000..5f82254f --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Juno backports. Do not use this number for new +# Kilo work. New Kilo work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py new file mode 100644 index 00000000..5f82254f --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Juno backports. Do not use this number for new +# Kilo work. New Kilo work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py new file mode 100644 index 00000000..5f82254f --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Juno backports. Do not use this number for new +# Kilo work. New Kilo work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py new file mode 100644 index 00000000..5f82254f --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is a placeholder for Juno backports. Do not use this number for new +# Kilo work. New Kilo work starts after all the placeholders. + + +def upgrade(migrate_engine): + pass + + +def downgrade(migration_engine): + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py new file mode 100644 index 00000000..bb8ef9f6 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.common.sql import migration_helpers + +_PROJECT_TABLE_NAME = 'project' +_PARENT_ID_COLUMN_NAME = 'parent_id' + + +def list_constraints(project_table): + constraints = [{'table': project_table, + 'fk_column': _PARENT_ID_COLUMN_NAME, + 'ref_column': project_table.c.id}] + + return constraints + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) + parent_id = sql.Column(_PARENT_ID_COLUMN_NAME, sql.String(64), + nullable=True) + project_table.create_column(parent_id) + + if migrate_engine.name == 'sqlite': + return + migration_helpers.add_constraints(list_constraints(project_table)) + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) + + # SQLite does not support constraints, and querying the constraints + # raises an exception + if migrate_engine.name != 'sqlite': + migration_helpers.remove_constraints(list_constraints(project_table)) + + project_table.drop_column(_PARENT_ID_COLUMN_NAME) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py new file mode 100644 index 00000000..5a33486c --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy + +from keystone.common.sql import migration_helpers + + +def list_constraints(migrate_engine): + meta = sqlalchemy.MetaData() + meta.bind = migrate_engine + assignment_table = sqlalchemy.Table('assignment', meta, autoload=True) + role_table = sqlalchemy.Table('role', meta, autoload=True) + + constraints = [{'table': assignment_table, + 'fk_column': 'role_id', + 'ref_column': role_table.c.id}] + return constraints + + +def upgrade(migrate_engine): + # SQLite does not support constraints, and querying the constraints + # raises an exception + if migrate_engine.name == 'sqlite': + return + migration_helpers.remove_constraints(list_constraints(migrate_engine)) + + +def downgrade(migrate_engine): + if migrate_engine.name == 'sqlite': + return + migration_helpers.add_constraints(list_constraints(migrate_engine)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py new file mode 100644 index 00000000..109a8412 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +_REGION_TABLE_NAME = 'region' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) + region_table.drop_column('url') + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True) + url_column = sql.Column('url', sql.String(255), nullable=True) + region_table.create_column(url_column) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py new file mode 100644 index 00000000..bca00902 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy + +from keystone.common.sql import migration_helpers + + +def list_constraints(migrate_engine): + meta = sqlalchemy.MetaData() + meta.bind = migrate_engine + user_table = sqlalchemy.Table('user', meta, autoload=True) + group_table = sqlalchemy.Table('group', meta, autoload=True) + domain_table = sqlalchemy.Table('domain', meta, autoload=True) + + constraints = [{'table': user_table, + 'fk_column': 'domain_id', + 'ref_column': domain_table.c.id}, + {'table': group_table, + 'fk_column': 'domain_id', + 'ref_column': domain_table.c.id}] + return constraints + + +def upgrade(migrate_engine): + # SQLite does not support constraints, and querying the constraints + # raises an exception + if migrate_engine.name == 'sqlite': + return + migration_helpers.remove_constraints(list_constraints(migrate_engine)) + + +def downgrade(migrate_engine): + if migrate_engine.name == 'sqlite': + return + migration_helpers.add_constraints(list_constraints(migrate_engine)) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py new file mode 100644 index 00000000..fd8717d2 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py @@ -0,0 +1,55 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.common import sql as ks_sql + +WHITELIST_TABLE = 'whitelisted_config' +SENSITIVE_TABLE = 'sensitive_config' + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + whitelist_table = sql.Table( + WHITELIST_TABLE, + meta, + sql.Column('domain_id', sql.String(64), primary_key=True), + sql.Column('group', sql.String(255), primary_key=True), + sql.Column('option', sql.String(255), primary_key=True), + sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + whitelist_table.create(migrate_engine, checkfirst=True) + + sensitive_table = sql.Table( + SENSITIVE_TABLE, + meta, + sql.Column('domain_id', sql.String(64), primary_key=True), + sql.Column('group', sql.String(255), primary_key=True), + sql.Column('option', sql.String(255), primary_key=True), + sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + sensitive_table.create(migrate_engine, checkfirst=True) + + +def downgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + table = sql.Table(WHITELIST_TABLE, meta, autoload=True) + table.drop(migrate_engine, checkfirst=True) + table = sql.Table(SENSITIVE_TABLE, meta, autoload=True) + table.drop(migrate_engine, checkfirst=True) diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py new file mode 100644 index 00000000..3feadc53 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_serialization import jsonutils +import sqlalchemy as sql + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + service_table = sql.Table('service', meta, autoload=True) + services = list(service_table.select().execute()) + + for service in services: + extra_dict = jsonutils.loads(service.extra) + # Skip records where service is not null + if extra_dict.get('name') is not None: + continue + # Default the name to empty string + extra_dict['name'] = '' + new_values = { + 'extra': jsonutils.dumps(extra_dict), + } + f = service_table.c.id == service.id + update = service_table.update().where(f).values(new_values) + migrate_engine.execute(update) + + +def downgrade(migration_engine): + # The upgrade fixes the data inconsistency for the service name, + # it defaults the value to empty string. There is no necessity + # to revert it. + pass diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/__init__.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/__init__.py diff --git a/keystone-moon/keystone/common/sql/migration_helpers.py b/keystone-moon/keystone/common/sql/migration_helpers.py new file mode 100644 index 00000000..86932995 --- /dev/null +++ b/keystone-moon/keystone/common/sql/migration_helpers.py @@ -0,0 +1,258 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import migrate +from migrate import exceptions +from oslo_config import cfg +from oslo_db.sqlalchemy import migration +from oslo_serialization import jsonutils +from oslo_utils import importutils +import six +import sqlalchemy + +from keystone.common import sql +from keystone.common.sql import migrate_repo +from keystone import contrib +from keystone import exception +from keystone.i18n import _ + + +CONF = cfg.CONF +DEFAULT_EXTENSIONS = ['endpoint_filter', + 'endpoint_policy', + 'federation', + 'oauth1', + 'revoke', + ] + + +def get_default_domain(): + # Return the reference used for the default domain structure during + # sql migrations. + return { + 'id': CONF.identity.default_domain_id, + 'name': 'Default', + 'enabled': True, + 'extra': jsonutils.dumps({'description': 'Owns users and tenants ' + '(i.e. projects) available ' + 'on Identity API v2.'})} + + +# Different RDBMSs use different schemes for naming the Foreign Key +# Constraints. SQLAlchemy does not yet attempt to determine the name +# for the constraint, and instead attempts to deduce it from the column. +# This fails on MySQL. +def get_constraints_names(table, column_name): + fkeys = [fk.name for fk in table.constraints + if (isinstance(fk, sqlalchemy.ForeignKeyConstraint) and + column_name in fk.columns)] + return fkeys + + +# remove_constraints and add_constraints both accept a list of dictionaries +# that contain: +# {'table': a sqlalchemy table. The constraint is added to dropped from +# this table. +# 'fk_column': the name of a column on the above table, The constraint +# is added to or dropped from this column +# 'ref_column':a sqlalchemy column object. This is the reference column +# for the constraint. +def remove_constraints(constraints): + for constraint_def in constraints: + constraint_names = get_constraints_names(constraint_def['table'], + constraint_def['fk_column']) + for constraint_name in constraint_names: + migrate.ForeignKeyConstraint( + columns=[getattr(constraint_def['table'].c, + constraint_def['fk_column'])], + refcolumns=[constraint_def['ref_column']], + name=constraint_name).drop() + + +def add_constraints(constraints): + for constraint_def in constraints: + + if constraint_def['table'].kwargs.get('mysql_engine') == 'MyISAM': + # Don't try to create constraint when using MyISAM because it's + # not supported. + continue + + ref_col = constraint_def['ref_column'] + ref_engine = ref_col.table.kwargs.get('mysql_engine') + if ref_engine == 'MyISAM': + # Don't try to create constraint when using MyISAM because it's + # not supported. + continue + + migrate.ForeignKeyConstraint( + columns=[getattr(constraint_def['table'].c, + constraint_def['fk_column'])], + refcolumns=[constraint_def['ref_column']]).create() + + +def rename_tables_with_constraints(renames, constraints, engine): + """Renames tables with foreign key constraints. + + Tables are renamed after first removing constraints. The constraints are + replaced after the rename is complete. + + This works on databases that don't support renaming tables that have + constraints on them (DB2). + + `renames` is a dict, mapping {'to_table_name': from_table, ...} + """ + + if engine.name != 'sqlite': + # Sqlite doesn't support constraints, so nothing to remove. + remove_constraints(constraints) + + for to_table_name in renames: + from_table = renames[to_table_name] + from_table.rename(to_table_name) + + if engine != 'sqlite': + add_constraints(constraints) + + +def find_migrate_repo(package=None, repo_name='migrate_repo'): + package = package or sql + path = os.path.abspath(os.path.join( + os.path.dirname(package.__file__), repo_name)) + if os.path.isdir(path): + return path + raise exception.MigrationNotProvided(package.__name__, path) + + +def _sync_common_repo(version): + abs_path = find_migrate_repo() + init_version = migrate_repo.DB_INIT_VERSION + engine = sql.get_engine() + migration.db_sync(engine, abs_path, version=version, + init_version=init_version) + + +def _fix_federation_tables(engine): + """Fix the identity_provider, federation_protocol and mapping tables + to be InnoDB and Charset UTF8. + + This function is to work around bug #1426334. This has occurred because + the original migration did not specify InnoDB and charset utf8. Due + to the sanity_check, a deployer can get wedged here and require manual + database changes to fix. + """ + # NOTE(marco-fargetta) This is a workaround to "fix" that tables only + # if we're under MySQL + if engine.name == 'mysql': + # * Disable any check for the foreign keys because they prevent the + # alter table to execute + engine.execute("SET foreign_key_checks = 0") + # * Make the tables using InnoDB engine + engine.execute("ALTER TABLE identity_provider Engine=InnoDB") + engine.execute("ALTER TABLE federation_protocol Engine=InnoDB") + engine.execute("ALTER TABLE mapping Engine=InnoDB") + # * Make the tables using utf8 encoding + engine.execute("ALTER TABLE identity_provider " + "CONVERT TO CHARACTER SET utf8") + engine.execute("ALTER TABLE federation_protocol " + "CONVERT TO CHARACTER SET utf8") + engine.execute("ALTER TABLE mapping CONVERT TO CHARACTER SET utf8") + # * Revert the foreign keys check back + engine.execute("SET foreign_key_checks = 1") + + +def _sync_extension_repo(extension, version): + init_version = 0 + engine = sql.get_engine() + + try: + package_name = '.'.join((contrib.__name__, extension)) + package = importutils.import_module(package_name) + except ImportError: + raise ImportError(_("%s extension does not exist.") + % package_name) + try: + abs_path = find_migrate_repo(package) + try: + migration.db_version_control(sql.get_engine(), abs_path) + # Register the repo with the version control API + # If it already knows about the repo, it will throw + # an exception that we can safely ignore + except exceptions.DatabaseAlreadyControlledError: + pass + except exception.MigrationNotProvided as e: + print(e) + sys.exit(1) + try: + migration.db_sync(engine, abs_path, version=version, + init_version=init_version) + except ValueError: + # NOTE(marco-fargetta): ValueError is raised from the sanity check ( + # verifies that tables are utf8 under mysql). The federation_protocol, + # identity_provider and mapping tables were not initially built with + # InnoDB and utf8 as part of the table arguments when the migration + # was initially created. Bug #1426334 is a scenario where the deployer + # can get wedged, unable to upgrade or downgrade. + # This is a workaround to "fix" those tables if we're under MySQL and + # the version is before the 6 because before the tables were introduced + # before and patched when migration 5 was available + if engine.name == 'mysql' and \ + int(six.text_type(get_db_version(extension))) < 6: + _fix_federation_tables(engine) + # The migration is applied again after the fix + migration.db_sync(engine, abs_path, version=version, + init_version=init_version) + else: + raise + + +def sync_database_to_version(extension=None, version=None): + if not extension: + _sync_common_repo(version) + # If version is greater than 0, it is for the common + # repository only, and only that will be synchronized. + if version is None: + for default_extension in DEFAULT_EXTENSIONS: + _sync_extension_repo(default_extension, version) + else: + _sync_extension_repo(extension, version) + + +def get_db_version(extension=None): + if not extension: + return migration.db_version(sql.get_engine(), find_migrate_repo(), + migrate_repo.DB_INIT_VERSION) + + try: + package_name = '.'.join((contrib.__name__, extension)) + package = importutils.import_module(package_name) + except ImportError: + raise ImportError(_("%s extension does not exist.") + % package_name) + + return migration.db_version( + sql.get_engine(), find_migrate_repo(package), 0) + + +def print_db_version(extension=None): + try: + db_version = get_db_version(extension=extension) + print(db_version) + except exception.MigrationNotProvided as e: + print(e) + sys.exit(1) |