migrate moon code from github to opnfv

Change-Id: Ice53e368fd1114d56a75271aa9f2e598e3eba604
Signed-off-by: WuKong <rebirthmonkey@gmail.com>

4 files changed, 618 insertions, 0 deletions

diff --git a/keystone-moon/keystone/catalog/backends/__init__.py b/keystone-moon/keystone/catalog/backends/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone-moon/keystone/catalog/backends/__init__.py
diff --git a/keystone-moon/keystone/catalog/backends/kvs.py b/keystone-moon/keystone/catalog/backends/kvs.py
new file mode 100644
index 00000000..30a121d8
--- /dev/null
+++ b/keystone-moon/keystone/catalog/backends/kvs.py
@@ -0,0 +1,154 @@
+# Copyright 2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from keystone import catalog
+from keystone.common import driver_hints
+from keystone.common import kvs
+
+
+class Catalog(kvs.Base, catalog.Driver):
+    # Public interface
+    def get_catalog(self, user_id, tenant_id):
+        return self.db.get('catalog-%s-%s' % (tenant_id, user_id))
+
+    # region crud
+
+    def _delete_child_regions(self, region_id, root_region_id):
+        """Delete all child regions.
+
+        Recursively delete any region that has the supplied region
+        as its parent.
+        """
+        children = [r for r in self.list_regions(driver_hints.Hints())
+                    if r['parent_region_id'] == region_id]
+        for child in children:
+            if child['id'] == root_region_id:
+                # Hit a circular region hierarchy
+                return
+            self._delete_child_regions(child['id'], root_region_id)
+            self._delete_region(child['id'])
+
+    def _check_parent_region(self, region_ref):
+        """Raise a NotFound if the parent region does not exist.
+
+        If the region_ref has a specified parent_region_id, check that
+        the parent exists, otherwise, raise a NotFound.
+        """
+        parent_region_id = region_ref.get('parent_region_id')
+        if parent_region_id is not None:
+            # This will raise NotFound if the parent doesn't exist,
+            # which is the behavior we want.
+            self.get_region(parent_region_id)
+
+    def create_region(self, region):
+        region_id = region['id']
+        region.setdefault('parent_region_id')
+        self._check_parent_region(region)
+        self.db.set('region-%s' % region_id, region)
+        region_list = set(self.db.get('region_list', []))
+        region_list.add(region_id)
+        self.db.set('region_list', list(region_list))
+        return region
+
+    def list_regions(self, hints):
+        return [self.get_region(x) for x in self.db.get('region_list', [])]
+
+    def get_region(self, region_id):
+        return self.db.get('region-%s' % region_id)
+
+    def update_region(self, region_id, region):
+        self._check_parent_region(region)
+        old_region = self.get_region(region_id)
+        old_region.update(region)
+        self._ensure_no_circle_in_hierarchical_regions(old_region)
+        self.db.set('region-%s' % region_id, old_region)
+        return old_region
+
+    def _delete_region(self, region_id):
+        self.db.delete('region-%s' % region_id)
+        region_list = set(self.db.get('region_list', []))
+        region_list.remove(region_id)
+        self.db.set('region_list', list(region_list))
+
+    def delete_region(self, region_id):
+        self._delete_child_regions(region_id, region_id)
+        self._delete_region(region_id)
+
+    # service crud
+
+    def create_service(self, service_id, service):
+        self.db.set('service-%s' % service_id, service)
+        service_list = set(self.db.get('service_list', []))
+        service_list.add(service_id)
+        self.db.set('service_list', list(service_list))
+        return service
+
+    def list_services(self, hints):
+        return [self.get_service(x) for x in self.db.get('service_list', [])]
+
+    def get_service(self, service_id):
+        return self.db.get('service-%s' % service_id)
+
+    def update_service(self, service_id, service):
+        old_service = self.get_service(service_id)
+        old_service.update(service)
+        self.db.set('service-%s' % service_id, old_service)
+        return old_service
+
+    def delete_service(self, service_id):
+        # delete referencing endpoints
+        for endpoint_id in self.db.get('endpoint_list', []):
+            if self.get_endpoint(endpoint_id)['service_id'] == service_id:
+                self.delete_endpoint(endpoint_id)
+
+        self.db.delete('service-%s' % service_id)
+        service_list = set(self.db.get('service_list', []))
+        service_list.remove(service_id)
+        self.db.set('service_list', list(service_list))
+
+    # endpoint crud
+
+    def create_endpoint(self, endpoint_id, endpoint):
+        self.db.set('endpoint-%s' % endpoint_id, endpoint)
+        endpoint_list = set(self.db.get('endpoint_list', []))
+        endpoint_list.add(endpoint_id)
+        self.db.set('endpoint_list', list(endpoint_list))
+        return endpoint
+
+    def list_endpoints(self, hints):
+        return [self.get_endpoint(x) for x in self.db.get('endpoint_list', [])]
+
+    def get_endpoint(self, endpoint_id):
+        return self.db.get('endpoint-%s' % endpoint_id)
+
+    def update_endpoint(self, endpoint_id, endpoint):
+        if endpoint.get('region_id') is not None:
+            self.get_region(endpoint['region_id'])
+
+        old_endpoint = self.get_endpoint(endpoint_id)
+        old_endpoint.update(endpoint)
+        self.db.set('endpoint-%s' % endpoint_id, old_endpoint)
+        return old_endpoint
+
+    def delete_endpoint(self, endpoint_id):
+        self.db.delete('endpoint-%s' % endpoint_id)
+        endpoint_list = set(self.db.get('endpoint_list', []))
+        endpoint_list.remove(endpoint_id)
+        self.db.set('endpoint_list', list(endpoint_list))
+
+    # Private interface
+    def _create_catalog(self, user_id, tenant_id, data):
+        self.db.set('catalog-%s-%s' % (tenant_id, user_id), data)
+        return data
diff --git a/keystone-moon/keystone/catalog/backends/sql.py b/keystone-moon/keystone/catalog/backends/sql.py
new file mode 100644
index 00000000..8ab82305
--- /dev/null
+++ b/keystone-moon/keystone/catalog/backends/sql.py
@@ -0,0 +1,337 @@
+# Copyright 2012 OpenStack Foundation
+# Copyright 2012 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import itertools
+
+from oslo_config import cfg
+import six
+import sqlalchemy
+from sqlalchemy.sql import true
+
+from keystone import catalog
+from keystone.catalog import core
+from keystone.common import sql
+from keystone import exception
+
+
+CONF = cfg.CONF
+
+
+class Region(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'region'
+    attributes = ['id', 'description', 'parent_region_id']
+    id = sql.Column(sql.String(255), primary_key=True)
+    description = sql.Column(sql.String(255), nullable=False)
+    # NOTE(jaypipes): Right now, using an adjacency list model for
+    #                 storing the hierarchy of regions is fine, since
+    #                 the API does not support any kind of querying for
+    #                 more complex hierarchical queries such as "get me only
+    #                 the regions that are subchildren of this region", etc.
+    #                 If, in the future, such queries are needed, then it
+    #                 would be possible to add in columns to this model for
+    #                 "left" and "right" and provide support for a nested set
+    #                 model.
+    parent_region_id = sql.Column(sql.String(255), nullable=True)
+
+    # TODO(jaypipes): I think it's absolutely stupid that every single model
+    #                 is required to have an "extra" column because of the
+    #                 DictBase in the keystone.common.sql.core module. Forcing
+    #                 tables to have pointless columns in the database is just
+    #                 bad. Remove all of this extra JSON blob stuff.
+    #                 See: https://bugs.launchpad.net/keystone/+bug/1265071
+    extra = sql.Column(sql.JsonBlob())
+    endpoints = sqlalchemy.orm.relationship("Endpoint", backref="region")
+
+
+class Service(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'service'
+    attributes = ['id', 'type', 'enabled']
+    id = sql.Column(sql.String(64), primary_key=True)
+    type = sql.Column(sql.String(255))
+    enabled = sql.Column(sql.Boolean, nullable=False, default=True,
+                         server_default=sqlalchemy.sql.expression.true())
+    extra = sql.Column(sql.JsonBlob())
+    endpoints = sqlalchemy.orm.relationship("Endpoint", backref="service")
+
+
+class Endpoint(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'endpoint'
+    attributes = ['id', 'interface', 'region_id', 'service_id', 'url',
+                  'legacy_endpoint_id', 'enabled']
+    id = sql.Column(sql.String(64), primary_key=True)
+    legacy_endpoint_id = sql.Column(sql.String(64))
+    interface = sql.Column(sql.String(8), nullable=False)
+    region_id = sql.Column(sql.String(255),
+                           sql.ForeignKey('region.id',
+                                          ondelete='RESTRICT'),
+                           nullable=True,
+                           default=None)
+    service_id = sql.Column(sql.String(64),
+                            sql.ForeignKey('service.id'),
+                            nullable=False)
+    url = sql.Column(sql.Text(), nullable=False)
+    enabled = sql.Column(sql.Boolean, nullable=False, default=True,
+                         server_default=sqlalchemy.sql.expression.true())
+    extra = sql.Column(sql.JsonBlob())
+
+
+class Catalog(catalog.Driver):
+    # Regions
+    def list_regions(self, hints):
+        session = sql.get_session()
+        regions = session.query(Region)
+        regions = sql.filter_limit_query(Region, regions, hints)
+        return [s.to_dict() for s in list(regions)]
+
+    def _get_region(self, session, region_id):
+        ref = session.query(Region).get(region_id)
+        if not ref:
+            raise exception.RegionNotFound(region_id=region_id)
+        return ref
+
+    def _delete_child_regions(self, session, region_id, root_region_id):
+        """Delete all child regions.
+
+        Recursively delete any region that has the supplied region
+        as its parent.
+        """
+        children = session.query(Region).filter_by(parent_region_id=region_id)
+        for child in children:
+            if child.id == root_region_id:
+                # Hit a circular region hierarchy
+                return
+            self._delete_child_regions(session, child.id, root_region_id)
+            session.delete(child)
+
+    def _check_parent_region(self, session, region_ref):
+        """Raise a NotFound if the parent region does not exist.
+
+        If the region_ref has a specified parent_region_id, check that
+        the parent exists, otherwise, raise a NotFound.
+        """
+        parent_region_id = region_ref.get('parent_region_id')
+        if parent_region_id is not None:
+            # This will raise NotFound if the parent doesn't exist,
+            # which is the behavior we want.
+            self._get_region(session, parent_region_id)
+
+    def _has_endpoints(self, session, region, root_region):
+        if region.endpoints is not None and len(region.endpoints) > 0:
+            return True
+
+        q = session.query(Region)
+        q = q.filter_by(parent_region_id=region.id)
+        for child in q.all():
+            if child.id == root_region.id:
+                # Hit a circular region hierarchy
+                return False
+            if self._has_endpoints(session, child, root_region):
+                return True
+        return False
+
+    def get_region(self, region_id):
+        session = sql.get_session()
+        return self._get_region(session, region_id).to_dict()
+
+    def delete_region(self, region_id):
+        session = sql.get_session()
+        with session.begin():
+            ref = self._get_region(session, region_id)
+            if self._has_endpoints(session, ref, ref):
+                raise exception.RegionDeletionError(region_id=region_id)
+            self._delete_child_regions(session, region_id, region_id)
+            session.delete(ref)
+
+    @sql.handle_conflicts(conflict_type='region')
+    def create_region(self, region_ref):
+        session = sql.get_session()
+        with session.begin():
+            self._check_parent_region(session, region_ref)
+            region = Region.from_dict(region_ref)
+            session.add(region)
+        return region.to_dict()
+
+    def update_region(self, region_id, region_ref):
+        session = sql.get_session()
+        with session.begin():
+            self._check_parent_region(session, region_ref)
+            ref = self._get_region(session, region_id)
+            old_dict = ref.to_dict()
+            old_dict.update(region_ref)
+            self._ensure_no_circle_in_hierarchical_regions(old_dict)
+            new_region = Region.from_dict(old_dict)
+            for attr in Region.attributes:
+                if attr != 'id':
+                    setattr(ref, attr, getattr(new_region, attr))
+        return ref.to_dict()
+
+    # Services
+    @sql.truncated
+    def list_services(self, hints):
+        session = sql.get_session()
+        services = session.query(Service)
+        services = sql.filter_limit_query(Service, services, hints)
+        return [s.to_dict() for s in list(services)]
+
+    def _get_service(self, session, service_id):
+        ref = session.query(Service).get(service_id)
+        if not ref:
+            raise exception.ServiceNotFound(service_id=service_id)
+        return ref
+
+    def get_service(self, service_id):
+        session = sql.get_session()
+        return self._get_service(session, service_id).to_dict()
+
+    def delete_service(self, service_id):
+        session = sql.get_session()
+        with session.begin():
+            ref = self._get_service(session, service_id)
+            session.query(Endpoint).filter_by(service_id=service_id).delete()
+            session.delete(ref)
+
+    def create_service(self, service_id, service_ref):
+        session = sql.get_session()
+        with session.begin():
+            service = Service.from_dict(service_ref)
+            session.add(service)
+        return service.to_dict()
+
+    def update_service(self, service_id, service_ref):
+        session = sql.get_session()
+        with session.begin():
+            ref = self._get_service(session, service_id)
+            old_dict = ref.to_dict()
+            old_dict.update(service_ref)
+            new_service = Service.from_dict(old_dict)
+            for attr in Service.attributes:
+                if attr != 'id':
+                    setattr(ref, attr, getattr(new_service, attr))
+            ref.extra = new_service.extra
+        return ref.to_dict()
+
+    # Endpoints
+    def create_endpoint(self, endpoint_id, endpoint_ref):
+        session = sql.get_session()
+        new_endpoint = Endpoint.from_dict(endpoint_ref)
+
+        with session.begin():
+            session.add(new_endpoint)
+        return new_endpoint.to_dict()
+
+    def delete_endpoint(self, endpoint_id):
+        session = sql.get_session()
+        with session.begin():
+            ref = self._get_endpoint(session, endpoint_id)
+            session.delete(ref)
+
+    def _get_endpoint(self, session, endpoint_id):
+        try:
+            return session.query(Endpoint).filter_by(id=endpoint_id).one()
+        except sql.NotFound:
+            raise exception.EndpointNotFound(endpoint_id=endpoint_id)
+
+    def get_endpoint(self, endpoint_id):
+        session = sql.get_session()
+        return self._get_endpoint(session, endpoint_id).to_dict()
+
+    @sql.truncated
+    def list_endpoints(self, hints):
+        session = sql.get_session()
+        endpoints = session.query(Endpoint)
+        endpoints = sql.filter_limit_query(Endpoint, endpoints, hints)
+        return [e.to_dict() for e in list(endpoints)]
+
+    def update_endpoint(self, endpoint_id, endpoint_ref):
+        session = sql.get_session()
+
+        with session.begin():
+            ref = self._get_endpoint(session, endpoint_id)
+            old_dict = ref.to_dict()
+            old_dict.update(endpoint_ref)
+            new_endpoint = Endpoint.from_dict(old_dict)
+            for attr in Endpoint.attributes:
+                if attr != 'id':
+                    setattr(ref, attr, getattr(new_endpoint, attr))
+            ref.extra = new_endpoint.extra
+        return ref.to_dict()
+
+    def get_catalog(self, user_id, tenant_id):
+        substitutions = dict(
+            itertools.chain(six.iteritems(CONF),
+                            six.iteritems(CONF.eventlet_server)))
+        substitutions.update({'tenant_id': tenant_id, 'user_id': user_id})
+
+        session = sql.get_session()
+        endpoints = (session.query(Endpoint).
+                     options(sql.joinedload(Endpoint.service)).
+                     filter(Endpoint.enabled == true()).all())
+
+        catalog = {}
+
+        for endpoint in endpoints:
+            if not endpoint.service['enabled']:
+                continue
+            try:
+                url = core.format_url(endpoint['url'], substitutions)
+            except exception.MalformedEndpoint:
+                continue  # this failure is already logged in format_url()
+
+            region = endpoint['region_id']
+            service_type = endpoint.service['type']
+            default_service = {
+                'id': endpoint['id'],
+                'name': endpoint.service.extra.get('name', ''),
+                'publicURL': ''
+            }
+            catalog.setdefault(region, {})
+            catalog[region].setdefault(service_type, default_service)
+            interface_url = '%sURL' % endpoint['interface']
+            catalog[region][service_type][interface_url] = url
+
+        return catalog
+
+    def get_v3_catalog(self, user_id, tenant_id):
+        d = dict(
+            itertools.chain(six.iteritems(CONF),
+                            six.iteritems(CONF.eventlet_server)))
+        d.update({'tenant_id': tenant_id,
+                  'user_id': user_id})
+
+        session = sql.get_session()
+        services = (session.query(Service).filter(Service.enabled == true()).
+                    options(sql.joinedload(Service.endpoints)).
+                    all())
+
+        def make_v3_endpoints(endpoints):
+            for endpoint in (ep.to_dict() for ep in endpoints if ep.enabled):
+                del endpoint['service_id']
+                del endpoint['legacy_endpoint_id']
+                del endpoint['enabled']
+                endpoint['region'] = endpoint['region_id']
+                try:
+                    endpoint['url'] = core.format_url(endpoint['url'], d)
+                except exception.MalformedEndpoint:
+                    continue  # this failure is already logged in format_url()
+
+                yield endpoint
+
+        def make_v3_service(svc):
+            eps = list(make_v3_endpoints(svc.endpoints))
+            service = {'endpoints': eps, 'id': svc.id, 'type': svc.type}
+            service['name'] = svc.extra.get('name', '')
+            return service
+
+        return [make_v3_service(svc) for svc in services]
diff --git a/keystone-moon/keystone/catalog/backends/templated.py b/keystone-moon/keystone/catalog/backends/templated.py
new file mode 100644
index 00000000..d3ee105d
--- /dev/null
+++ b/keystone-moon/keystone/catalog/backends/templated.py
@@ -0,0 +1,127 @@
+# Copyright 2012 OpenStack Foundationc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import itertools
+import os.path
+
+from oslo_config import cfg
+from oslo_log import log
+import six
+
+from keystone.catalog.backends import kvs
+from keystone.catalog import core
+from keystone import exception
+from keystone.i18n import _LC
+
+
+LOG = log.getLogger(__name__)
+
+CONF = cfg.CONF
+
+
+def parse_templates(template_lines):
+    o = {}
+    for line in template_lines:
+        if ' = ' not in line:
+            continue
+
+        k, v = line.strip().split(' = ')
+        if not k.startswith('catalog.'):
+            continue
+
+        parts = k.split('.')
+
+        region = parts[1]
+        # NOTE(termie): object-store insists on having a dash
+        service = parts[2].replace('_', '-')
+        key = parts[3]
+
+        region_ref = o.get(region, {})
+        service_ref = region_ref.get(service, {})
+        service_ref[key] = v
+
+        region_ref[service] = service_ref
+        o[region] = region_ref
+
+    return o
+
+
+class Catalog(kvs.Catalog):
+    """A backend that generates endpoints for the Catalog based on templates.
+
+    It is usually configured via config entries that look like:
+
+      catalog.$REGION.$SERVICE.$key = $value
+
+    and is stored in a similar looking hierarchy. Where a value can contain
+    values to be interpolated by standard python string interpolation that look
+    like (the % is replaced by a $ due to paste attempting to interpolate on
+    its own:
+
+      http://localhost:$(public_port)s/
+
+    When expanding the template it will pass in a dict made up of the conf
+    instance plus a few additional key-values, notably tenant_id and user_id.
+
+    It does not care what the keys and values are but it is worth noting that
+    keystone_compat will expect certain keys to be there so that it can munge
+    them into the output format keystone expects. These keys are:
+
+      name - the name of the service, most likely repeated for all services of
+             the same type, across regions.
+
+      adminURL - the url of the admin endpoint
+
+      publicURL - the url of the public endpoint
+
+      internalURL - the url of the internal endpoint
+
+    """
+
+    def __init__(self, templates=None):
+        super(Catalog, self).__init__()
+        if templates:
+            self.templates = templates
+        else:
+            template_file = CONF.catalog.template_file
+            if not os.path.exists(template_file):
+                template_file = CONF.find_file(template_file)
+            self._load_templates(template_file)
+
+    def _load_templates(self, template_file):
+        try:
+            self.templates = parse_templates(open(template_file))
+        except IOError:
+            LOG.critical(_LC('Unable to open template file %s'), template_file)
+            raise
+
+    def get_catalog(self, user_id, tenant_id):
+        substitutions = dict(
+            itertools.chain(six.iteritems(CONF),
+                            six.iteritems(CONF.eventlet_server)))
+        substitutions.update({'tenant_id': tenant_id, 'user_id': user_id})
+
+        catalog = {}
+        for region, region_ref in six.iteritems(self.templates):
+            catalog[region] = {}
+            for service, service_ref in six.iteritems(region_ref):
+                service_data = {}
+                try:
+                    for k, v in six.iteritems(service_ref):
+                        service_data[k] = core.format_url(v, substitutions)
+                except exception.MalformedEndpoint:
+                    continue  # this failure is already logged in format_url()
+                catalog[region][service] = service_data
+
+        return catalog