aboutsummaryrefslogtreecommitdiffstats
path: root/keystone-moon/keystone/common/sql
diff options
context:
space:
mode:
authorWuKong <rebirthmonkey@gmail.com>2015-06-30 18:47:29 +0200
committerWuKong <rebirthmonkey@gmail.com>2015-06-30 18:47:29 +0200
commitb8c756ecdd7cced1db4300935484e8c83701c82e (patch)
tree87e51107d82b217ede145de9d9d59e2100725bd7 /keystone-moon/keystone/common/sql
parentc304c773bae68fb854ed9eab8fb35c4ef17cf136 (diff)
migrate moon code from github to opnfv
Change-Id: Ice53e368fd1114d56a75271aa9f2e598e3eba604 Signed-off-by: WuKong <rebirthmonkey@gmail.com>
Diffstat (limited to 'keystone-moon/keystone/common/sql')
-rw-r--r--keystone-moon/keystone/common/sql/__init__.py15
-rw-r--r--keystone-moon/keystone/common/sql/core.py431
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/README4
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/__init__.py17
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/manage.py5
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/migrate.cfg25
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py279
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py25
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py25
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py25
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py25
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py25
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py49
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py49
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py34
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py156
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py35
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py35
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py22
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py22
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py22
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py22
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py22
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py54
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py41
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py32
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py45
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py55
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py43
-rw-r--r--keystone-moon/keystone/common/sql/migrate_repo/versions/__init__.py0
-rw-r--r--keystone-moon/keystone/common/sql/migration_helpers.py258
31 files changed, 1897 insertions, 0 deletions
diff --git a/keystone-moon/keystone/common/sql/__init__.py b/keystone-moon/keystone/common/sql/__init__.py
new file mode 100644
index 00000000..84e0fb83
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from keystone.common.sql.core import * # noqa
diff --git a/keystone-moon/keystone/common/sql/core.py b/keystone-moon/keystone/common/sql/core.py
new file mode 100644
index 00000000..bf168701
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/core.py
@@ -0,0 +1,431 @@
+# Copyright 2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""SQL backends for the various services.
+
+Before using this module, call initialize(). This has to be done before
+CONF() because it sets up configuration options.
+
+"""
+import contextlib
+import functools
+
+from oslo_config import cfg
+from oslo_db import exception as db_exception
+from oslo_db import options as db_options
+from oslo_db.sqlalchemy import models
+from oslo_db.sqlalchemy import session as db_session
+from oslo_log import log
+from oslo_serialization import jsonutils
+import six
+import sqlalchemy as sql
+from sqlalchemy.ext import declarative
+from sqlalchemy.orm.attributes import flag_modified, InstrumentedAttribute
+from sqlalchemy import types as sql_types
+
+from keystone.common import utils
+from keystone import exception
+from keystone.i18n import _
+
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+ModelBase = declarative.declarative_base()
+
+
+# For exporting to other modules
+Column = sql.Column
+Index = sql.Index
+String = sql.String
+Integer = sql.Integer
+Enum = sql.Enum
+ForeignKey = sql.ForeignKey
+DateTime = sql.DateTime
+IntegrityError = sql.exc.IntegrityError
+DBDuplicateEntry = db_exception.DBDuplicateEntry
+OperationalError = sql.exc.OperationalError
+NotFound = sql.orm.exc.NoResultFound
+Boolean = sql.Boolean
+Text = sql.Text
+UniqueConstraint = sql.UniqueConstraint
+PrimaryKeyConstraint = sql.PrimaryKeyConstraint
+joinedload = sql.orm.joinedload
+# Suppress flake8's unused import warning for flag_modified:
+flag_modified = flag_modified
+
+
+def initialize():
+ """Initialize the module."""
+
+ db_options.set_defaults(
+ CONF,
+ connection="sqlite:///keystone.db")
+
+
+def initialize_decorator(init):
+ """Ensure that the length of string field do not exceed the limit.
+
+ This decorator check the initialize arguments, to make sure the
+ length of string field do not exceed the length limit, or raise a
+ 'StringLengthExceeded' exception.
+
+ Use decorator instead of inheritance, because the metaclass will
+ check the __tablename__, primary key columns, etc. at the class
+ definition.
+
+ """
+ def initialize(self, *args, **kwargs):
+ cls = type(self)
+ for k, v in kwargs.items():
+ if hasattr(cls, k):
+ attr = getattr(cls, k)
+ if isinstance(attr, InstrumentedAttribute):
+ column = attr.property.columns[0]
+ if isinstance(column.type, String):
+ if not isinstance(v, six.text_type):
+ v = six.text_type(v)
+ if column.type.length and column.type.length < len(v):
+ raise exception.StringLengthExceeded(
+ string=v, type=k, length=column.type.length)
+
+ init(self, *args, **kwargs)
+ return initialize
+
+ModelBase.__init__ = initialize_decorator(ModelBase.__init__)
+
+
+# Special Fields
+class JsonBlob(sql_types.TypeDecorator):
+
+ impl = sql.Text
+
+ def process_bind_param(self, value, dialect):
+ return jsonutils.dumps(value)
+
+ def process_result_value(self, value, dialect):
+ return jsonutils.loads(value)
+
+
+class DictBase(models.ModelBase):
+ attributes = []
+
+ @classmethod
+ def from_dict(cls, d):
+ new_d = d.copy()
+
+ new_d['extra'] = {k: new_d.pop(k) for k in six.iterkeys(d)
+ if k not in cls.attributes and k != 'extra'}
+
+ return cls(**new_d)
+
+ def to_dict(self, include_extra_dict=False):
+ """Returns the model's attributes as a dictionary.
+
+ If include_extra_dict is True, 'extra' attributes are literally
+ included in the resulting dictionary twice, for backwards-compatibility
+ with a broken implementation.
+
+ """
+ d = self.extra.copy()
+ for attr in self.__class__.attributes:
+ d[attr] = getattr(self, attr)
+
+ if include_extra_dict:
+ d['extra'] = self.extra.copy()
+
+ return d
+
+ def __getitem__(self, key):
+ if key in self.extra:
+ return self.extra[key]
+ return getattr(self, key)
+
+
+class ModelDictMixin(object):
+
+ @classmethod
+ def from_dict(cls, d):
+ """Returns a model instance from a dictionary."""
+ return cls(**d)
+
+ def to_dict(self):
+ """Returns the model's attributes as a dictionary."""
+ names = (column.name for column in self.__table__.columns)
+ return {name: getattr(self, name) for name in names}
+
+
+_engine_facade = None
+
+
+def _get_engine_facade():
+ global _engine_facade
+
+ if not _engine_facade:
+ _engine_facade = db_session.EngineFacade.from_config(CONF)
+
+ return _engine_facade
+
+
+def cleanup():
+ global _engine_facade
+
+ _engine_facade = None
+
+
+def get_engine():
+ return _get_engine_facade().get_engine()
+
+
+def get_session(expire_on_commit=False):
+ return _get_engine_facade().get_session(expire_on_commit=expire_on_commit)
+
+
+@contextlib.contextmanager
+def transaction(expire_on_commit=False):
+ """Return a SQLAlchemy session in a scoped transaction."""
+ session = get_session(expire_on_commit=expire_on_commit)
+ with session.begin():
+ yield session
+
+
+def truncated(f):
+ """Ensure list truncation is detected in Driver list entity methods.
+
+ This is designed to wrap and sql Driver list_{entity} methods in order to
+ calculate if the resultant list has been truncated. Provided a limit dict
+ is found in the hints list, we increment the limit by one so as to ask the
+ wrapped function for one more entity than the limit, and then once the list
+ has been generated, we check to see if the original limit has been
+ exceeded, in which case we truncate back to that limit and set the
+ 'truncated' boolean to 'true' in the hints limit dict.
+
+ """
+ @functools.wraps(f)
+ def wrapper(self, hints, *args, **kwargs):
+ if not hasattr(hints, 'limit'):
+ raise exception.UnexpectedError(
+ _('Cannot truncate a driver call without hints list as '
+ 'first parameter after self '))
+
+ if hints.limit is None:
+ return f(self, hints, *args, **kwargs)
+
+ # A limit is set, so ask for one more entry than we need
+ list_limit = hints.limit['limit']
+ hints.set_limit(list_limit + 1)
+ ref_list = f(self, hints, *args, **kwargs)
+
+ # If we got more than the original limit then trim back the list and
+ # mark it truncated. In both cases, make sure we set the limit back
+ # to its original value.
+ if len(ref_list) > list_limit:
+ hints.set_limit(list_limit, truncated=True)
+ return ref_list[:list_limit]
+ else:
+ hints.set_limit(list_limit)
+ return ref_list
+ return wrapper
+
+
+def _filter(model, query, hints):
+ """Applies filtering to a query.
+
+ :param model: the table model in question
+ :param query: query to apply filters to
+ :param hints: contains the list of filters yet to be satisfied.
+ Any filters satisfied here will be removed so that
+ the caller will know if any filters remain.
+
+ :returns query: query, updated with any filters satisfied
+
+ """
+ def inexact_filter(model, query, filter_, satisfied_filters, hints):
+ """Applies an inexact filter to a query.
+
+ :param model: the table model in question
+ :param query: query to apply filters to
+ :param filter_: the dict that describes this filter
+ :param satisfied_filters: a cumulative list of satisfied filters, to
+ which filter_ will be added if it is
+ satisfied.
+ :param hints: contains the list of filters yet to be satisfied.
+
+ :returns query: query updated to add any inexact filters we could
+ satisfy
+
+ """
+ column_attr = getattr(model, filter_['name'])
+
+ # TODO(henry-nash): Sqlalchemy 0.7 defaults to case insensitivity
+ # so once we find a way of changing that (maybe on a call-by-call
+ # basis), we can add support for the case sensitive versions of
+ # the filters below. For now, these case sensitive versions will
+ # be handled at the controller level.
+
+ if filter_['case_sensitive']:
+ return query
+
+ if filter_['comparator'] == 'contains':
+ query_term = column_attr.ilike('%%%s%%' % filter_['value'])
+ elif filter_['comparator'] == 'startswith':
+ query_term = column_attr.ilike('%s%%' % filter_['value'])
+ elif filter_['comparator'] == 'endswith':
+ query_term = column_attr.ilike('%%%s' % filter_['value'])
+ else:
+ # It's a filter we don't understand, so let the caller
+ # work out if they need to do something with it.
+ return query
+
+ satisfied_filters.append(filter_)
+ return query.filter(query_term)
+
+ def exact_filter(
+ model, filter_, satisfied_filters, cumulative_filter_dict, hints):
+ """Applies an exact filter to a query.
+
+ :param model: the table model in question
+ :param filter_: the dict that describes this filter
+ :param satisfied_filters: a cumulative list of satisfied filters, to
+ which filter_ will be added if it is
+ satisfied.
+ :param cumulative_filter_dict: a dict that describes the set of
+ exact filters built up so far
+ :param hints: contains the list of filters yet to be satisfied.
+
+ :returns: updated cumulative dict
+
+ """
+ key = filter_['name']
+ if isinstance(getattr(model, key).property.columns[0].type,
+ sql.types.Boolean):
+ cumulative_filter_dict[key] = (
+ utils.attr_as_boolean(filter_['value']))
+ else:
+ cumulative_filter_dict[key] = filter_['value']
+ satisfied_filters.append(filter_)
+ return cumulative_filter_dict
+
+ filter_dict = {}
+ satisfied_filters = []
+ for filter_ in hints.filters:
+ if filter_['name'] not in model.attributes:
+ continue
+ if filter_['comparator'] == 'equals':
+ filter_dict = exact_filter(
+ model, filter_, satisfied_filters, filter_dict, hints)
+ else:
+ query = inexact_filter(
+ model, query, filter_, satisfied_filters, hints)
+
+ # Apply any exact filters we built up
+ if filter_dict:
+ query = query.filter_by(**filter_dict)
+
+ # Remove satisfied filters, then the caller will know remaining filters
+ for filter_ in satisfied_filters:
+ hints.filters.remove(filter_)
+
+ return query
+
+
+def _limit(query, hints):
+ """Applies a limit to a query.
+
+ :param query: query to apply filters to
+ :param hints: contains the list of filters and limit details.
+
+ :returns updated query
+
+ """
+ # NOTE(henry-nash): If we were to implement pagination, then we
+ # we would expand this method to support pagination and limiting.
+
+ # If we satisfied all the filters, set an upper limit if supplied
+ if hints.limit:
+ query = query.limit(hints.limit['limit'])
+ return query
+
+
+def filter_limit_query(model, query, hints):
+ """Applies filtering and limit to a query.
+
+ :param model: table model
+ :param query: query to apply filters to
+ :param hints: contains the list of filters and limit details. This may
+ be None, indicating that there are no filters or limits
+ to be applied. If it's not None, then any filters
+ satisfied here will be removed so that the caller will
+ know if any filters remain.
+
+ :returns: updated query
+
+ """
+ if hints is None:
+ return query
+
+ # First try and satisfy any filters
+ query = _filter(model, query, hints)
+
+ # NOTE(henry-nash): Any unsatisfied filters will have been left in
+ # the hints list for the controller to handle. We can only try and
+ # limit here if all the filters are already satisfied since, if not,
+ # doing so might mess up the final results. If there are still
+ # unsatisfied filters, we have to leave any limiting to the controller
+ # as well.
+
+ if not hints.filters:
+ return _limit(query, hints)
+ else:
+ return query
+
+
+def handle_conflicts(conflict_type='object'):
+ """Converts select sqlalchemy exceptions into HTTP 409 Conflict."""
+ _conflict_msg = 'Conflict %(conflict_type)s: %(details)s'
+
+ def decorator(method):
+ @functools.wraps(method)
+ def wrapper(*args, **kwargs):
+ try:
+ return method(*args, **kwargs)
+ except db_exception.DBDuplicateEntry as e:
+ # LOG the exception for debug purposes, do not send the
+ # exception details out with the raised Conflict exception
+ # as it can contain raw SQL.
+ LOG.debug(_conflict_msg, {'conflict_type': conflict_type,
+ 'details': six.text_type(e)})
+ raise exception.Conflict(type=conflict_type,
+ details=_('Duplicate Entry'))
+ except db_exception.DBError as e:
+ # TODO(blk-u): inspecting inner_exception breaks encapsulation;
+ # oslo_db should provide exception we need.
+ if isinstance(e.inner_exception, IntegrityError):
+ # LOG the exception for debug purposes, do not send the
+ # exception details out with the raised Conflict exception
+ # as it can contain raw SQL.
+ LOG.debug(_conflict_msg, {'conflict_type': conflict_type,
+ 'details': six.text_type(e)})
+ # NOTE(morganfainberg): This is really a case where the SQL
+ # failed to store the data. This is not something that the
+ # user has done wrong. Example would be a ForeignKey is
+ # missing; the code that is executed before reaching the
+ # SQL writing to the DB should catch the issue.
+ raise exception.UnexpectedError(
+ _('An unexpected error occurred when trying to '
+ 'store %s') % conflict_type)
+ raise
+
+ return wrapper
+ return decorator
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/README b/keystone-moon/keystone/common/sql/migrate_repo/README
new file mode 100644
index 00000000..6218f8ca
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/README
@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/__init__.py b/keystone-moon/keystone/common/sql/migrate_repo/__init__.py
new file mode 100644
index 00000000..f73dfc12
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2014 Mirantis.inc
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+DB_INIT_VERSION = 43
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/manage.py b/keystone-moon/keystone/common/sql/migrate_repo/manage.py
new file mode 100644
index 00000000..39fa3892
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/manage.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+ main(debug='False')
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/migrate.cfg b/keystone-moon/keystone/common/sql/migrate_repo/migrate.cfg
new file mode 100644
index 00000000..db531bb4
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/migrate.cfg
@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=keystone
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py
new file mode 100644
index 00000000..6f326ecf
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/044_icehouse.py
@@ -0,0 +1,279 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import migrate
+from oslo_config import cfg
+from oslo_log import log
+import sqlalchemy as sql
+from sqlalchemy import orm
+
+from keystone.assignment.backends import sql as assignment_sql
+from keystone.common import sql as ks_sql
+from keystone.common.sql import migration_helpers
+
+
+LOG = log.getLogger(__name__)
+CONF = cfg.CONF
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ if migrate_engine.name == 'mysql':
+ # In Folsom we explicitly converted migrate_version to UTF8.
+ migrate_engine.execute(
+ 'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8')
+ # Set default DB charset to UTF8.
+ migrate_engine.execute(
+ 'ALTER DATABASE %s DEFAULT CHARACTER SET utf8' %
+ migrate_engine.url.database)
+
+ credential = sql.Table(
+ 'credential', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('user_id', sql.String(length=64), nullable=False),
+ sql.Column('project_id', sql.String(length=64)),
+ sql.Column('blob', ks_sql.JsonBlob, nullable=False),
+ sql.Column('type', sql.String(length=255), nullable=False),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ domain = sql.Table(
+ 'domain', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('name', sql.String(length=64), nullable=False),
+ sql.Column('enabled', sql.Boolean, default=True, nullable=False),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ endpoint = sql.Table(
+ 'endpoint', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('legacy_endpoint_id', sql.String(length=64)),
+ sql.Column('interface', sql.String(length=8), nullable=False),
+ sql.Column('region', sql.String(length=255)),
+ sql.Column('service_id', sql.String(length=64), nullable=False),
+ sql.Column('url', sql.Text, nullable=False),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ sql.Column('enabled', sql.Boolean, nullable=False, default=True,
+ server_default='1'),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ group = sql.Table(
+ 'group', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('domain_id', sql.String(length=64), nullable=False),
+ sql.Column('name', sql.String(length=64), nullable=False),
+ sql.Column('description', sql.Text),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ policy = sql.Table(
+ 'policy', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('type', sql.String(length=255), nullable=False),
+ sql.Column('blob', ks_sql.JsonBlob, nullable=False),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ project = sql.Table(
+ 'project', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('name', sql.String(length=64), nullable=False),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ sql.Column('description', sql.Text),
+ sql.Column('enabled', sql.Boolean),
+ sql.Column('domain_id', sql.String(length=64), nullable=False),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ role = sql.Table(
+ 'role', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('name', sql.String(length=255), nullable=False),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ service = sql.Table(
+ 'service', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('type', sql.String(length=255)),
+ sql.Column('enabled', sql.Boolean, nullable=False, default=True,
+ server_default='1'),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ token = sql.Table(
+ 'token', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('expires', sql.DateTime, default=None),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ sql.Column('valid', sql.Boolean, default=True, nullable=False),
+ sql.Column('trust_id', sql.String(length=64)),
+ sql.Column('user_id', sql.String(length=64)),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ trust = sql.Table(
+ 'trust', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('trustor_user_id', sql.String(length=64), nullable=False),
+ sql.Column('trustee_user_id', sql.String(length=64), nullable=False),
+ sql.Column('project_id', sql.String(length=64)),
+ sql.Column('impersonation', sql.Boolean, nullable=False),
+ sql.Column('deleted_at', sql.DateTime),
+ sql.Column('expires_at', sql.DateTime),
+ sql.Column('remaining_uses', sql.Integer, nullable=True),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ trust_role = sql.Table(
+ 'trust_role', meta,
+ sql.Column('trust_id', sql.String(length=64), primary_key=True,
+ nullable=False),
+ sql.Column('role_id', sql.String(length=64), primary_key=True,
+ nullable=False),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ user = sql.Table(
+ 'user', meta,
+ sql.Column('id', sql.String(length=64), primary_key=True),
+ sql.Column('name', sql.String(length=255), nullable=False),
+ sql.Column('extra', ks_sql.JsonBlob.impl),
+ sql.Column('password', sql.String(length=128)),
+ sql.Column('enabled', sql.Boolean),
+ sql.Column('domain_id', sql.String(length=64), nullable=False),
+ sql.Column('default_project_id', sql.String(length=64)),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ user_group_membership = sql.Table(
+ 'user_group_membership', meta,
+ sql.Column('user_id', sql.String(length=64), primary_key=True),
+ sql.Column('group_id', sql.String(length=64), primary_key=True),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ region = sql.Table(
+ 'region',
+ meta,
+ sql.Column('id', sql.String(64), primary_key=True),
+ sql.Column('description', sql.String(255), nullable=False),
+ sql.Column('parent_region_id', sql.String(64), nullable=True),
+ sql.Column('extra', sql.Text()),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ assignment = sql.Table(
+ 'assignment',
+ meta,
+ sql.Column('type', sql.Enum(
+ assignment_sql.AssignmentType.USER_PROJECT,
+ assignment_sql.AssignmentType.GROUP_PROJECT,
+ assignment_sql.AssignmentType.USER_DOMAIN,
+ assignment_sql.AssignmentType.GROUP_DOMAIN,
+ name='type'),
+ nullable=False),
+ sql.Column('actor_id', sql.String(64), nullable=False),
+ sql.Column('target_id', sql.String(64), nullable=False),
+ sql.Column('role_id', sql.String(64), nullable=False),
+ sql.Column('inherited', sql.Boolean, default=False, nullable=False),
+ sql.PrimaryKeyConstraint('type', 'actor_id', 'target_id', 'role_id'),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+
+ # create all tables
+ tables = [credential, domain, endpoint, group,
+ policy, project, role, service,
+ token, trust, trust_role, user,
+ user_group_membership, region, assignment]
+
+ for table in tables:
+ try:
+ table.create()
+ except Exception:
+ LOG.exception('Exception while creating table: %r', table)
+ raise
+
+ # Unique Constraints
+ migrate.UniqueConstraint(user.c.domain_id,
+ user.c.name,
+ name='ixu_user_name_domain_id').create()
+ migrate.UniqueConstraint(group.c.domain_id,
+ group.c.name,
+ name='ixu_group_name_domain_id').create()
+ migrate.UniqueConstraint(role.c.name,
+ name='ixu_role_name').create()
+ migrate.UniqueConstraint(project.c.domain_id,
+ project.c.name,
+ name='ixu_project_name_domain_id').create()
+ migrate.UniqueConstraint(domain.c.name,
+ name='ixu_domain_name').create()
+
+ # Indexes
+ sql.Index('ix_token_expires', token.c.expires).create()
+ sql.Index('ix_token_expires_valid', token.c.expires,
+ token.c.valid).create()
+
+ fkeys = [
+ {'columns': [endpoint.c.service_id],
+ 'references': [service.c.id]},
+
+ {'columns': [user_group_membership.c.group_id],
+ 'references': [group.c.id],
+ 'name': 'fk_user_group_membership_group_id'},
+
+ {'columns': [user_group_membership.c.user_id],
+ 'references':[user.c.id],
+ 'name': 'fk_user_group_membership_user_id'},
+
+ {'columns': [user.c.domain_id],
+ 'references': [domain.c.id],
+ 'name': 'fk_user_domain_id'},
+
+ {'columns': [group.c.domain_id],
+ 'references': [domain.c.id],
+ 'name': 'fk_group_domain_id'},
+
+ {'columns': [project.c.domain_id],
+ 'references': [domain.c.id],
+ 'name': 'fk_project_domain_id'},
+
+ {'columns': [assignment.c.role_id],
+ 'references': [role.c.id]}
+ ]
+
+ for fkey in fkeys:
+ migrate.ForeignKeyConstraint(columns=fkey['columns'],
+ refcolumns=fkey['references'],
+ name=fkey.get('name')).create()
+
+ # Create the default domain.
+ session = orm.sessionmaker(bind=migrate_engine)()
+ domain.insert(migration_helpers.get_default_domain()).execute()
+ session.commit()
+
+
+def downgrade(migrate_engine):
+ raise NotImplementedError('Downgrade to pre-Icehouse release db schema is '
+ 'unsupported.')
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py
new file mode 100644
index 00000000..b6f40719
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/045_placeholder.py
@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Icehouse backports. Do not use this number for new
+# Juno work. New Juno work starts after all the placeholders.
+#
+# See blueprint reserved-db-migrations-icehouse and the related discussion:
+# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py
new file mode 100644
index 00000000..b6f40719
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/046_placeholder.py
@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Icehouse backports. Do not use this number for new
+# Juno work. New Juno work starts after all the placeholders.
+#
+# See blueprint reserved-db-migrations-icehouse and the related discussion:
+# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py
new file mode 100644
index 00000000..b6f40719
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/047_placeholder.py
@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Icehouse backports. Do not use this number for new
+# Juno work. New Juno work starts after all the placeholders.
+#
+# See blueprint reserved-db-migrations-icehouse and the related discussion:
+# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py
new file mode 100644
index 00000000..b6f40719
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/048_placeholder.py
@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Icehouse backports. Do not use this number for new
+# Juno work. New Juno work starts after all the placeholders.
+#
+# See blueprint reserved-db-migrations-icehouse and the related discussion:
+# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py
new file mode 100644
index 00000000..b6f40719
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/049_placeholder.py
@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Icehouse backports. Do not use this number for new
+# Juno work. New Juno work starts after all the placeholders.
+#
+# See blueprint reserved-db-migrations-icehouse and the related discussion:
+# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py
new file mode 100644
index 00000000..535a0944
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/050_fk_consistent_indexes.py
@@ -0,0 +1,49 @@
+# Copyright 2014 Mirantis.inc
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sa
+
+
+def upgrade(migrate_engine):
+
+ if migrate_engine.name == 'mysql':
+ meta = sa.MetaData(bind=migrate_engine)
+ endpoint = sa.Table('endpoint', meta, autoload=True)
+
+ # NOTE(i159): MySQL requires indexes on referencing columns, and those
+ # indexes create automatically. That those indexes will have different
+ # names, depending on version of MySQL used. We shoud make this naming
+ # consistent, by reverting index name to a consistent condition.
+ if any(i for i in endpoint.indexes if
+ i.columns.keys() == ['service_id'] and i.name != 'service_id'):
+ # NOTE(i159): by this action will be made re-creation of an index
+ # with the new name. This can be considered as renaming under the
+ # MySQL rules.
+ sa.Index('service_id', endpoint.c.service_id).create()
+
+ user_group_membership = sa.Table('user_group_membership',
+ meta, autoload=True)
+
+ if any(i for i in user_group_membership.indexes if
+ i.columns.keys() == ['group_id'] and i.name != 'group_id'):
+ sa.Index('group_id', user_group_membership.c.group_id).create()
+
+
+def downgrade(migrate_engine):
+ # NOTE(i159): index exists only in MySQL schemas, and got an inconsistent
+ # name only when MySQL 5.5 renamed it after re-creation
+ # (during migrations). So we just fixed inconsistency, there is no
+ # necessity to revert it.
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py
new file mode 100644
index 00000000..074fbb63
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/051_add_id_mapping.py
@@ -0,0 +1,49 @@
+# Copyright 2014 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+from keystone.identity.mapping_backends import mapping
+
+
+MAPPING_TABLE = 'id_mapping'
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ mapping_table = sql.Table(
+ MAPPING_TABLE,
+ meta,
+ sql.Column('public_id', sql.String(64), primary_key=True),
+ sql.Column('domain_id', sql.String(64), nullable=False),
+ sql.Column('local_id', sql.String(64), nullable=False),
+ sql.Column('entity_type', sql.Enum(
+ mapping.EntityType.USER,
+ mapping.EntityType.GROUP,
+ name='entity_type'),
+ nullable=False),
+ sql.UniqueConstraint('domain_id', 'local_id', 'entity_type'),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+ mapping_table.create(migrate_engine, checkfirst=True)
+
+
+def downgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ assignment = sql.Table(MAPPING_TABLE, meta, autoload=True)
+ assignment.drop(migrate_engine, checkfirst=True)
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py
new file mode 100644
index 00000000..9f1fd9f0
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/052_add_auth_url_to_region.py
@@ -0,0 +1,34 @@
+# Copyright 2014 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+_REGION_TABLE_NAME = 'region'
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True)
+ url_column = sql.Column('url', sql.String(255), nullable=True)
+ region_table.create_column(url_column)
+
+
+def downgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True)
+ region_table.drop_column('url')
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py
new file mode 100644
index 00000000..6dc0004f
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/053_endpoint_to_region_association.py
@@ -0,0 +1,156 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""Migrated the endpoint 'region' column to 'region_id.
+
+In addition to the rename, the new column is made a foreign key to the
+respective 'region' in the region table, ensuring that we auto-create
+any regions that are missing. Further, since the old region column
+was 255 chars, and the id column in the region table is 64 chars, the size
+of the id column in the region table is increased to match.
+
+To Upgrade:
+
+
+Region Table
+
+Increase the size of the if column in the region table
+
+Endpoint Table
+
+a. Add the endpoint region_id column, that is a foreign key to the region table
+b. For each endpoint
+ i. Ensure there is matching region in region table, and if not, create it
+ ii. Assign the id to the region_id column
+c. Remove the column region
+
+
+To Downgrade:
+
+Endpoint Table
+
+a. Add back in the region column
+b. For each endpoint
+ i. Copy the region_id column to the region column
+c. Remove the column region_id
+
+Region Table
+
+Decrease the size of the id column in the region table, making sure that
+we don't get classing primary keys.
+
+"""
+
+import migrate
+import six
+import sqlalchemy as sql
+from sqlalchemy.orm import sessionmaker
+
+
+def _migrate_to_region_id(migrate_engine, region_table, endpoint_table):
+ endpoints = list(endpoint_table.select().execute())
+
+ for endpoint in endpoints:
+ if endpoint.region is None:
+ continue
+
+ region = list(region_table.select(
+ whereclause=region_table.c.id == endpoint.region).execute())
+ if len(region) == 1:
+ region_id = region[0].id
+ else:
+ region_id = endpoint.region
+ region = {'id': region_id,
+ 'description': '',
+ 'extra': '{}'}
+ session = sessionmaker(bind=migrate_engine)()
+ region_table.insert(region).execute()
+ session.commit()
+
+ new_values = {'region_id': region_id}
+ f = endpoint_table.c.id == endpoint.id
+ update = endpoint_table.update().where(f).values(new_values)
+ migrate_engine.execute(update)
+
+ migrate.ForeignKeyConstraint(
+ columns=[endpoint_table.c.region_id],
+ refcolumns=[region_table.c.id],
+ name='fk_endpoint_region_id').create()
+
+
+def _migrate_to_region(migrate_engine, region_table, endpoint_table):
+ endpoints = list(endpoint_table.select().execute())
+
+ for endpoint in endpoints:
+ new_values = {'region': endpoint.region_id}
+ f = endpoint_table.c.id == endpoint.id
+ update = endpoint_table.update().where(f).values(new_values)
+ migrate_engine.execute(update)
+
+ if 'sqlite' != migrate_engine.name:
+ migrate.ForeignKeyConstraint(
+ columns=[endpoint_table.c.region_id],
+ refcolumns=[region_table.c.id],
+ name='fk_endpoint_region_id').drop()
+ endpoint_table.c.region_id.drop()
+
+
+def _prepare_regions_for_id_truncation(migrate_engine, region_table):
+ """Ensure there are no IDs that are bigger than 64 chars.
+
+ The size of the id and parent_id fields where increased from 64 to 255
+ during the upgrade. On downgrade we have to make sure that the ids can
+ fit in the new column size. For rows with ids greater than this, we have
+ no choice but to dump them.
+
+ """
+ for region in list(region_table.select().execute()):
+ if (len(six.text_type(region.id)) > 64 or
+ len(six.text_type(region.parent_region_id)) > 64):
+ delete = region_table.delete(region_table.c.id == region.id)
+ migrate_engine.execute(delete)
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ region_table = sql.Table('region', meta, autoload=True)
+ region_table.c.id.alter(type=sql.String(length=255))
+ region_table.c.parent_region_id.alter(type=sql.String(length=255))
+ endpoint_table = sql.Table('endpoint', meta, autoload=True)
+ region_id_column = sql.Column('region_id',
+ sql.String(length=255), nullable=True)
+ region_id_column.create(endpoint_table)
+
+ _migrate_to_region_id(migrate_engine, region_table, endpoint_table)
+
+ endpoint_table.c.region.drop()
+
+
+def downgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ region_table = sql.Table('region', meta, autoload=True)
+ endpoint_table = sql.Table('endpoint', meta, autoload=True)
+ region_column = sql.Column('region', sql.String(length=255))
+ region_column.create(endpoint_table)
+
+ _migrate_to_region(migrate_engine, region_table, endpoint_table)
+ _prepare_regions_for_id_truncation(migrate_engine, region_table)
+
+ region_table.c.id.alter(type=sql.String(length=64))
+ region_table.c.parent_region_id.alter(type=sql.String(length=64))
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py
new file mode 100644
index 00000000..33b13b7d
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/054_add_actor_id_index.py
@@ -0,0 +1,35 @@
+# Copyright 2014 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+ASSIGNMENT_TABLE = 'assignment'
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True)
+ idx = sql.Index('ix_actor_id', assignment.c.actor_id)
+ idx.create(migrate_engine)
+
+
+def downgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ assignment = sql.Table(ASSIGNMENT_TABLE, meta, autoload=True)
+ idx = sql.Index('ix_actor_id', assignment.c.actor_id)
+ idx.drop(migrate_engine)
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py
new file mode 100644
index 00000000..1cfddd3f
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/055_add_indexes_to_token_table.py
@@ -0,0 +1,35 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Add indexes to `user_id` and `trust_id` columns for the `token` table."""
+
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ token = sql.Table('token', meta, autoload=True)
+
+ sql.Index('ix_token_user_id', token.c.user_id).create()
+ sql.Index('ix_token_trust_id', token.c.trust_id).create()
+
+
+def downgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ token = sql.Table('token', meta, autoload=True)
+
+ sql.Index('ix_token_user_id', token.c.user_id).drop()
+ sql.Index('ix_token_trust_id', token.c.trust_id).drop()
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py
new file mode 100644
index 00000000..5f82254f
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/056_placeholder.py
@@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Juno backports. Do not use this number for new
+# Kilo work. New Kilo work starts after all the placeholders.
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py
new file mode 100644
index 00000000..5f82254f
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/057_placeholder.py
@@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Juno backports. Do not use this number for new
+# Kilo work. New Kilo work starts after all the placeholders.
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py
new file mode 100644
index 00000000..5f82254f
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/058_placeholder.py
@@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Juno backports. Do not use this number for new
+# Kilo work. New Kilo work starts after all the placeholders.
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py
new file mode 100644
index 00000000..5f82254f
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/059_placeholder.py
@@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Juno backports. Do not use this number for new
+# Kilo work. New Kilo work starts after all the placeholders.
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py
new file mode 100644
index 00000000..5f82254f
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/060_placeholder.py
@@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is a placeholder for Juno backports. Do not use this number for new
+# Kilo work. New Kilo work starts after all the placeholders.
+
+
+def upgrade(migrate_engine):
+ pass
+
+
+def downgrade(migration_engine):
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py
new file mode 100644
index 00000000..bb8ef9f6
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/061_add_parent_project.py
@@ -0,0 +1,54 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+from keystone.common.sql import migration_helpers
+
+_PROJECT_TABLE_NAME = 'project'
+_PARENT_ID_COLUMN_NAME = 'parent_id'
+
+
+def list_constraints(project_table):
+ constraints = [{'table': project_table,
+ 'fk_column': _PARENT_ID_COLUMN_NAME,
+ 'ref_column': project_table.c.id}]
+
+ return constraints
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True)
+ parent_id = sql.Column(_PARENT_ID_COLUMN_NAME, sql.String(64),
+ nullable=True)
+ project_table.create_column(parent_id)
+
+ if migrate_engine.name == 'sqlite':
+ return
+ migration_helpers.add_constraints(list_constraints(project_table))
+
+
+def downgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True)
+
+ # SQLite does not support constraints, and querying the constraints
+ # raises an exception
+ if migrate_engine.name != 'sqlite':
+ migration_helpers.remove_constraints(list_constraints(project_table))
+
+ project_table.drop_column(_PARENT_ID_COLUMN_NAME)
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py
new file mode 100644
index 00000000..5a33486c
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/062_drop_assignment_role_fk.py
@@ -0,0 +1,41 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy
+
+from keystone.common.sql import migration_helpers
+
+
+def list_constraints(migrate_engine):
+ meta = sqlalchemy.MetaData()
+ meta.bind = migrate_engine
+ assignment_table = sqlalchemy.Table('assignment', meta, autoload=True)
+ role_table = sqlalchemy.Table('role', meta, autoload=True)
+
+ constraints = [{'table': assignment_table,
+ 'fk_column': 'role_id',
+ 'ref_column': role_table.c.id}]
+ return constraints
+
+
+def upgrade(migrate_engine):
+ # SQLite does not support constraints, and querying the constraints
+ # raises an exception
+ if migrate_engine.name == 'sqlite':
+ return
+ migration_helpers.remove_constraints(list_constraints(migrate_engine))
+
+
+def downgrade(migrate_engine):
+ if migrate_engine.name == 'sqlite':
+ return
+ migration_helpers.add_constraints(list_constraints(migrate_engine))
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py
new file mode 100644
index 00000000..109a8412
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/063_drop_region_auth_url.py
@@ -0,0 +1,32 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+_REGION_TABLE_NAME = 'region'
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True)
+ region_table.drop_column('url')
+
+
+def downgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ region_table = sql.Table(_REGION_TABLE_NAME, meta, autoload=True)
+ url_column = sql.Column('url', sql.String(255), nullable=True)
+ region_table.create_column(url_column)
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py
new file mode 100644
index 00000000..bca00902
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/064_drop_user_and_group_fk.py
@@ -0,0 +1,45 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy
+
+from keystone.common.sql import migration_helpers
+
+
+def list_constraints(migrate_engine):
+ meta = sqlalchemy.MetaData()
+ meta.bind = migrate_engine
+ user_table = sqlalchemy.Table('user', meta, autoload=True)
+ group_table = sqlalchemy.Table('group', meta, autoload=True)
+ domain_table = sqlalchemy.Table('domain', meta, autoload=True)
+
+ constraints = [{'table': user_table,
+ 'fk_column': 'domain_id',
+ 'ref_column': domain_table.c.id},
+ {'table': group_table,
+ 'fk_column': 'domain_id',
+ 'ref_column': domain_table.c.id}]
+ return constraints
+
+
+def upgrade(migrate_engine):
+ # SQLite does not support constraints, and querying the constraints
+ # raises an exception
+ if migrate_engine.name == 'sqlite':
+ return
+ migration_helpers.remove_constraints(list_constraints(migrate_engine))
+
+
+def downgrade(migrate_engine):
+ if migrate_engine.name == 'sqlite':
+ return
+ migration_helpers.add_constraints(list_constraints(migrate_engine))
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py
new file mode 100644
index 00000000..fd8717d2
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/065_add_domain_config.py
@@ -0,0 +1,55 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+from keystone.common import sql as ks_sql
+
+WHITELIST_TABLE = 'whitelisted_config'
+SENSITIVE_TABLE = 'sensitive_config'
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ whitelist_table = sql.Table(
+ WHITELIST_TABLE,
+ meta,
+ sql.Column('domain_id', sql.String(64), primary_key=True),
+ sql.Column('group', sql.String(255), primary_key=True),
+ sql.Column('option', sql.String(255), primary_key=True),
+ sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+ whitelist_table.create(migrate_engine, checkfirst=True)
+
+ sensitive_table = sql.Table(
+ SENSITIVE_TABLE,
+ meta,
+ sql.Column('domain_id', sql.String(64), primary_key=True),
+ sql.Column('group', sql.String(255), primary_key=True),
+ sql.Column('option', sql.String(255), primary_key=True),
+ sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8')
+ sensitive_table.create(migrate_engine, checkfirst=True)
+
+
+def downgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ table = sql.Table(WHITELIST_TABLE, meta, autoload=True)
+ table.drop(migrate_engine, checkfirst=True)
+ table = sql.Table(SENSITIVE_TABLE, meta, autoload=True)
+ table.drop(migrate_engine, checkfirst=True)
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py
new file mode 100644
index 00000000..3feadc53
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/066_fixup_service_name_value.py
@@ -0,0 +1,43 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_serialization import jsonutils
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+ meta = sql.MetaData()
+ meta.bind = migrate_engine
+
+ service_table = sql.Table('service', meta, autoload=True)
+ services = list(service_table.select().execute())
+
+ for service in services:
+ extra_dict = jsonutils.loads(service.extra)
+ # Skip records where service is not null
+ if extra_dict.get('name') is not None:
+ continue
+ # Default the name to empty string
+ extra_dict['name'] = ''
+ new_values = {
+ 'extra': jsonutils.dumps(extra_dict),
+ }
+ f = service_table.c.id == service.id
+ update = service_table.update().where(f).values(new_values)
+ migrate_engine.execute(update)
+
+
+def downgrade(migration_engine):
+ # The upgrade fixes the data inconsistency for the service name,
+ # it defaults the value to empty string. There is no necessity
+ # to revert it.
+ pass
diff --git a/keystone-moon/keystone/common/sql/migrate_repo/versions/__init__.py b/keystone-moon/keystone/common/sql/migrate_repo/versions/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migrate_repo/versions/__init__.py
diff --git a/keystone-moon/keystone/common/sql/migration_helpers.py b/keystone-moon/keystone/common/sql/migration_helpers.py
new file mode 100644
index 00000000..86932995
--- /dev/null
+++ b/keystone-moon/keystone/common/sql/migration_helpers.py
@@ -0,0 +1,258 @@
+# Copyright 2013 OpenStack Foundation
+# Copyright 2013 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+
+import migrate
+from migrate import exceptions
+from oslo_config import cfg
+from oslo_db.sqlalchemy import migration
+from oslo_serialization import jsonutils
+from oslo_utils import importutils
+import six
+import sqlalchemy
+
+from keystone.common import sql
+from keystone.common.sql import migrate_repo
+from keystone import contrib
+from keystone import exception
+from keystone.i18n import _
+
+
+CONF = cfg.CONF
+DEFAULT_EXTENSIONS = ['endpoint_filter',
+ 'endpoint_policy',
+ 'federation',
+ 'oauth1',
+ 'revoke',
+ ]
+
+
+def get_default_domain():
+ # Return the reference used for the default domain structure during
+ # sql migrations.
+ return {
+ 'id': CONF.identity.default_domain_id,
+ 'name': 'Default',
+ 'enabled': True,
+ 'extra': jsonutils.dumps({'description': 'Owns users and tenants '
+ '(i.e. projects) available '
+ 'on Identity API v2.'})}
+
+
+# Different RDBMSs use different schemes for naming the Foreign Key
+# Constraints. SQLAlchemy does not yet attempt to determine the name
+# for the constraint, and instead attempts to deduce it from the column.
+# This fails on MySQL.
+def get_constraints_names(table, column_name):
+ fkeys = [fk.name for fk in table.constraints
+ if (isinstance(fk, sqlalchemy.ForeignKeyConstraint) and
+ column_name in fk.columns)]
+ return fkeys
+
+
+# remove_constraints and add_constraints both accept a list of dictionaries
+# that contain:
+# {'table': a sqlalchemy table. The constraint is added to dropped from
+# this table.
+# 'fk_column': the name of a column on the above table, The constraint
+# is added to or dropped from this column
+# 'ref_column':a sqlalchemy column object. This is the reference column
+# for the constraint.
+def remove_constraints(constraints):
+ for constraint_def in constraints:
+ constraint_names = get_constraints_names(constraint_def['table'],
+ constraint_def['fk_column'])
+ for constraint_name in constraint_names:
+ migrate.ForeignKeyConstraint(
+ columns=[getattr(constraint_def['table'].c,
+ constraint_def['fk_column'])],
+ refcolumns=[constraint_def['ref_column']],
+ name=constraint_name).drop()
+
+
+def add_constraints(constraints):
+ for constraint_def in constraints:
+
+ if constraint_def['table'].kwargs.get('mysql_engine') == 'MyISAM':
+ # Don't try to create constraint when using MyISAM because it's
+ # not supported.
+ continue
+
+ ref_col = constraint_def['ref_column']
+ ref_engine = ref_col.table.kwargs.get('mysql_engine')
+ if ref_engine == 'MyISAM':
+ # Don't try to create constraint when using MyISAM because it's
+ # not supported.
+ continue
+
+ migrate.ForeignKeyConstraint(
+ columns=[getattr(constraint_def['table'].c,
+ constraint_def['fk_column'])],
+ refcolumns=[constraint_def['ref_column']]).create()
+
+
+def rename_tables_with_constraints(renames, constraints, engine):
+ """Renames tables with foreign key constraints.
+
+ Tables are renamed after first removing constraints. The constraints are
+ replaced after the rename is complete.
+
+ This works on databases that don't support renaming tables that have
+ constraints on them (DB2).
+
+ `renames` is a dict, mapping {'to_table_name': from_table, ...}
+ """
+
+ if engine.name != 'sqlite':
+ # Sqlite doesn't support constraints, so nothing to remove.
+ remove_constraints(constraints)
+
+ for to_table_name in renames:
+ from_table = renames[to_table_name]
+ from_table.rename(to_table_name)
+
+ if engine != 'sqlite':
+ add_constraints(constraints)
+
+
+def find_migrate_repo(package=None, repo_name='migrate_repo'):
+ package = package or sql
+ path = os.path.abspath(os.path.join(
+ os.path.dirname(package.__file__), repo_name))
+ if os.path.isdir(path):
+ return path
+ raise exception.MigrationNotProvided(package.__name__, path)
+
+
+def _sync_common_repo(version):
+ abs_path = find_migrate_repo()
+ init_version = migrate_repo.DB_INIT_VERSION
+ engine = sql.get_engine()
+ migration.db_sync(engine, abs_path, version=version,
+ init_version=init_version)
+
+
+def _fix_federation_tables(engine):
+ """Fix the identity_provider, federation_protocol and mapping tables
+ to be InnoDB and Charset UTF8.
+
+ This function is to work around bug #1426334. This has occurred because
+ the original migration did not specify InnoDB and charset utf8. Due
+ to the sanity_check, a deployer can get wedged here and require manual
+ database changes to fix.
+ """
+ # NOTE(marco-fargetta) This is a workaround to "fix" that tables only
+ # if we're under MySQL
+ if engine.name == 'mysql':
+ # * Disable any check for the foreign keys because they prevent the
+ # alter table to execute
+ engine.execute("SET foreign_key_checks = 0")
+ # * Make the tables using InnoDB engine
+ engine.execute("ALTER TABLE identity_provider Engine=InnoDB")
+ engine.execute("ALTER TABLE federation_protocol Engine=InnoDB")
+ engine.execute("ALTER TABLE mapping Engine=InnoDB")
+ # * Make the tables using utf8 encoding
+ engine.execute("ALTER TABLE identity_provider "
+ "CONVERT TO CHARACTER SET utf8")
+ engine.execute("ALTER TABLE federation_protocol "
+ "CONVERT TO CHARACTER SET utf8")
+ engine.execute("ALTER TABLE mapping CONVERT TO CHARACTER SET utf8")
+ # * Revert the foreign keys check back
+ engine.execute("SET foreign_key_checks = 1")
+
+
+def _sync_extension_repo(extension, version):
+ init_version = 0
+ engine = sql.get_engine()
+
+ try:
+ package_name = '.'.join((contrib.__name__, extension))
+ package = importutils.import_module(package_name)
+ except ImportError:
+ raise ImportError(_("%s extension does not exist.")
+ % package_name)
+ try:
+ abs_path = find_migrate_repo(package)
+ try:
+ migration.db_version_control(sql.get_engine(), abs_path)
+ # Register the repo with the version control API
+ # If it already knows about the repo, it will throw
+ # an exception that we can safely ignore
+ except exceptions.DatabaseAlreadyControlledError:
+ pass
+ except exception.MigrationNotProvided as e:
+ print(e)
+ sys.exit(1)
+ try:
+ migration.db_sync(engine, abs_path, version=version,
+ init_version=init_version)
+ except ValueError:
+ # NOTE(marco-fargetta): ValueError is raised from the sanity check (
+ # verifies that tables are utf8 under mysql). The federation_protocol,
+ # identity_provider and mapping tables were not initially built with
+ # InnoDB and utf8 as part of the table arguments when the migration
+ # was initially created. Bug #1426334 is a scenario where the deployer
+ # can get wedged, unable to upgrade or downgrade.
+ # This is a workaround to "fix" those tables if we're under MySQL and
+ # the version is before the 6 because before the tables were introduced
+ # before and patched when migration 5 was available
+ if engine.name == 'mysql' and \
+ int(six.text_type(get_db_version(extension))) < 6:
+ _fix_federation_tables(engine)
+ # The migration is applied again after the fix
+ migration.db_sync(engine, abs_path, version=version,
+ init_version=init_version)
+ else:
+ raise
+
+
+def sync_database_to_version(extension=None, version=None):
+ if not extension:
+ _sync_common_repo(version)
+ # If version is greater than 0, it is for the common
+ # repository only, and only that will be synchronized.
+ if version is None:
+ for default_extension in DEFAULT_EXTENSIONS:
+ _sync_extension_repo(default_extension, version)
+ else:
+ _sync_extension_repo(extension, version)
+
+
+def get_db_version(extension=None):
+ if not extension:
+ return migration.db_version(sql.get_engine(), find_migrate_repo(),
+ migrate_repo.DB_INIT_VERSION)
+
+ try:
+ package_name = '.'.join((contrib.__name__, extension))
+ package = importutils.import_module(package_name)
+ except ImportError:
+ raise ImportError(_("%s extension does not exist.")
+ % package_name)
+
+ return migration.db_version(
+ sql.get_engine(), find_migrate_repo(package), 0)
+
+
+def print_db_version(extension=None):
+ try:
+ db_version = get_db_version(extension=extension)
+ print(db_version)
+ except exception.MigrationNotProvided as e:
+ print(e)
+ sys.exit(1)