diff options
Diffstat (limited to 'keystone-moon/keystone/common/cache')
| -rw-r--r-- | keystone-moon/keystone/common/cache/__init__.py | 15 | ||||
| -rw-r--r-- | keystone-moon/keystone/common/cache/_memcache_pool.py | 233 | ||||
| -rw-r--r-- | keystone-moon/keystone/common/cache/backends/__init__.py | 0 | ||||
| -rw-r--r-- | keystone-moon/keystone/common/cache/backends/memcache_pool.py | 61 | ||||
| -rw-r--r-- | keystone-moon/keystone/common/cache/backends/mongo.py | 557 | ||||
| -rw-r--r-- | keystone-moon/keystone/common/cache/backends/noop.py | 49 | ||||
| -rw-r--r-- | keystone-moon/keystone/common/cache/core.py | 308 |
7 files changed, 1223 insertions, 0 deletions
diff --git a/keystone-moon/keystone/common/cache/__init__.py b/keystone-moon/keystone/common/cache/__init__.py new file mode 100644 index 00000000..49502399 --- /dev/null +++ b/keystone-moon/keystone/common/cache/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2013 Metacloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystone.common.cache.core import * # noqa diff --git a/keystone-moon/keystone/common/cache/_memcache_pool.py b/keystone-moon/keystone/common/cache/_memcache_pool.py new file mode 100644 index 00000000..b15332db --- /dev/null +++ b/keystone-moon/keystone/common/cache/_memcache_pool.py @@ -0,0 +1,233 @@ +# Copyright 2014 Mirantis Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Thread-safe connection pool for python-memcached.""" + +# NOTE(yorik-sar): this file is copied between keystone and keystonemiddleware +# and should be kept in sync until we can use external library for this. + +import collections +import contextlib +import itertools +import logging +import threading +import time + +import memcache +from oslo_log import log +from six.moves import queue + +from keystone import exception +from keystone.i18n import _ + + +LOG = log.getLogger(__name__) + +# This 'class' is taken from http://stackoverflow.com/a/22520633/238308 +# Don't inherit client from threading.local so that we can reuse clients in +# different threads +_MemcacheClient = type('_MemcacheClient', (object,), + dict(memcache.Client.__dict__)) + +_PoolItem = collections.namedtuple('_PoolItem', ['ttl', 'connection']) + + +class ConnectionPool(queue.Queue): + """Base connection pool class + + This class implements the basic connection pool logic as an abstract base + class. + """ + def __init__(self, maxsize, unused_timeout, conn_get_timeout=None): + """Initialize the connection pool. + + :param maxsize: maximum number of client connections for the pool + :type maxsize: int + :param unused_timeout: idle time to live for unused clients (in + seconds). If a client connection object has been + in the pool and idle for longer than the + unused_timeout, it will be reaped. This is to + ensure resources are released as utilization + goes down. + :type unused_timeout: int + :param conn_get_timeout: maximum time in seconds to wait for a + connection. If set to `None` timeout is + indefinite. + :type conn_get_timeout: int + """ + # super() cannot be used here because Queue in stdlib is an + # old-style class + queue.Queue.__init__(self, maxsize) + self._unused_timeout = unused_timeout + self._connection_get_timeout = conn_get_timeout + self._acquired = 0 + + def _create_connection(self): + """Returns a connection instance. + + This is called when the pool needs another instance created. + + :returns: a new connection instance + + """ + raise NotImplementedError + + def _destroy_connection(self, conn): + """Destroy and cleanup a connection instance. + + This is called when the pool wishes to get rid of an existing + connection. This is the opportunity for a subclass to free up + resources and cleaup after itself. + + :param conn: the connection object to destroy + + """ + raise NotImplementedError + + def _debug_logger(self, msg, *args, **kwargs): + if LOG.isEnabledFor(logging.DEBUG): + thread_id = threading.current_thread().ident + args = (id(self), thread_id) + args + prefix = 'Memcached pool %s, thread %s: ' + LOG.debug(prefix + msg, *args, **kwargs) + + @contextlib.contextmanager + def acquire(self): + self._debug_logger('Acquiring connection') + try: + conn = self.get(timeout=self._connection_get_timeout) + except queue.Empty: + raise exception.UnexpectedError( + _('Unable to get a connection from pool id %(id)s after ' + '%(seconds)s seconds.') % + {'id': id(self), 'seconds': self._connection_get_timeout}) + self._debug_logger('Acquired connection %s', id(conn)) + try: + yield conn + finally: + self._debug_logger('Releasing connection %s', id(conn)) + self._drop_expired_connections() + try: + # super() cannot be used here because Queue in stdlib is an + # old-style class + queue.Queue.put(self, conn, block=False) + except queue.Full: + self._debug_logger('Reaping exceeding connection %s', id(conn)) + self._destroy_connection(conn) + + def _qsize(self): + if self.maxsize: + return self.maxsize - self._acquired + else: + # A value indicating there is always a free connection + # if maxsize is None or 0 + return 1 + + # NOTE(dstanek): stdlib and eventlet Queue implementations + # have different names for the qsize method. This ensures + # that we override both of them. + if not hasattr(queue.Queue, '_qsize'): + qsize = _qsize + + def _get(self): + if self.queue: + conn = self.queue.pop().connection + else: + conn = self._create_connection() + self._acquired += 1 + return conn + + def _drop_expired_connections(self): + """Drop all expired connections from the right end of the queue.""" + now = time.time() + while self.queue and self.queue[0].ttl < now: + conn = self.queue.popleft().connection + self._debug_logger('Reaping connection %s', id(conn)) + self._destroy_connection(conn) + + def _put(self, conn): + self.queue.append(_PoolItem( + ttl=time.time() + self._unused_timeout, + connection=conn, + )) + self._acquired -= 1 + + +class MemcacheClientPool(ConnectionPool): + def __init__(self, urls, arguments, **kwargs): + # super() cannot be used here because Queue in stdlib is an + # old-style class + ConnectionPool.__init__(self, **kwargs) + self.urls = urls + self._arguments = arguments + # NOTE(morganfainberg): The host objects expect an int for the + # deaduntil value. Initialize this at 0 for each host with 0 indicating + # the host is not dead. + self._hosts_deaduntil = [0] * len(urls) + + def _create_connection(self): + return _MemcacheClient(self.urls, **self._arguments) + + def _destroy_connection(self, conn): + conn.disconnect_all() + + def _get(self): + # super() cannot be used here because Queue in stdlib is an + # old-style class + conn = ConnectionPool._get(self) + try: + # Propagate host state known to us to this client's list + now = time.time() + for deaduntil, host in zip(self._hosts_deaduntil, conn.servers): + if deaduntil > now and host.deaduntil <= now: + host.mark_dead('propagating death mark from the pool') + host.deaduntil = deaduntil + except Exception: + # We need to be sure that connection doesn't leak from the pool. + # This code runs before we enter context manager's try-finally + # block, so we need to explicitly release it here. + # super() cannot be used here because Queue in stdlib is an + # old-style class + ConnectionPool._put(self, conn) + raise + return conn + + def _put(self, conn): + try: + # If this client found that one of the hosts is dead, mark it as + # such in our internal list + now = time.time() + for i, host in zip(itertools.count(), conn.servers): + deaduntil = self._hosts_deaduntil[i] + # Do nothing if we already know this host is dead + if deaduntil <= now: + if host.deaduntil > now: + self._hosts_deaduntil[i] = host.deaduntil + self._debug_logger( + 'Marked host %s dead until %s', + self.urls[i], host.deaduntil) + else: + self._hosts_deaduntil[i] = 0 + # If all hosts are dead we should forget that they're dead. This + # way we won't get completely shut off until dead_retry seconds + # pass, but will be checking servers as frequent as we can (over + # way smaller socket_timeout) + if all(deaduntil > now for deaduntil in self._hosts_deaduntil): + self._debug_logger('All hosts are dead. Marking them as live.') + self._hosts_deaduntil[:] = [0] * len(self._hosts_deaduntil) + finally: + # super() cannot be used here because Queue in stdlib is an + # old-style class + ConnectionPool._put(self, conn) diff --git a/keystone-moon/keystone/common/cache/backends/__init__.py b/keystone-moon/keystone/common/cache/backends/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/keystone-moon/keystone/common/cache/backends/__init__.py diff --git a/keystone-moon/keystone/common/cache/backends/memcache_pool.py b/keystone-moon/keystone/common/cache/backends/memcache_pool.py new file mode 100644 index 00000000..f3990b12 --- /dev/null +++ b/keystone-moon/keystone/common/cache/backends/memcache_pool.py @@ -0,0 +1,61 @@ +# Copyright 2014 Mirantis Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""dogpile.cache backend that uses Memcached connection pool""" + +import functools +import logging + +from dogpile.cache.backends import memcached as memcached_backend + +from keystone.common.cache import _memcache_pool + + +LOG = logging.getLogger(__name__) + + +# Helper to ease backend refactoring +class ClientProxy(object): + def __init__(self, client_pool): + self.client_pool = client_pool + + def _run_method(self, __name, *args, **kwargs): + with self.client_pool.acquire() as client: + return getattr(client, __name)(*args, **kwargs) + + def __getattr__(self, name): + return functools.partial(self._run_method, name) + + +class PooledMemcachedBackend(memcached_backend.MemcachedBackend): + # Composed from GenericMemcachedBackend's and MemcacheArgs's __init__ + def __init__(self, arguments): + super(PooledMemcachedBackend, self).__init__(arguments) + self.client_pool = _memcache_pool.MemcacheClientPool( + self.url, + arguments={ + 'dead_retry': arguments.get('dead_retry', 5 * 60), + 'socket_timeout': arguments.get('socket_timeout', 3), + }, + maxsize=arguments.get('pool_maxsize', 10), + unused_timeout=arguments.get('pool_unused_timeout', 60), + conn_get_timeout=arguments.get('pool_connection_get_timeout', 10), + ) + + # Since all methods in backend just call one of methods of client, this + # lets us avoid need to hack it too much + @property + def client(self): + return ClientProxy(self.client_pool) diff --git a/keystone-moon/keystone/common/cache/backends/mongo.py b/keystone-moon/keystone/common/cache/backends/mongo.py new file mode 100644 index 00000000..b5de9bc4 --- /dev/null +++ b/keystone-moon/keystone/common/cache/backends/mongo.py @@ -0,0 +1,557 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import datetime + +from dogpile.cache import api +from dogpile.cache import util as dp_util +from oslo_log import log +from oslo_utils import importutils +from oslo_utils import timeutils +import six + +from keystone import exception +from keystone.i18n import _, _LW + + +NO_VALUE = api.NO_VALUE +LOG = log.getLogger(__name__) + + +class MongoCacheBackend(api.CacheBackend): + """A MongoDB based caching backend implementing dogpile backend APIs. + + Arguments accepted in the arguments dictionary: + + :param db_hosts: string (required), hostname or IP address of the + MongoDB server instance. This can be a single MongoDB connection URI, + or a list of MongoDB connection URIs. + + :param db_name: string (required), the name of the database to be used. + + :param cache_collection: string (required), the name of collection to store + cached data. + *Note:* Different collection name can be provided if there is need to + create separate container (i.e. collection) for cache data. So region + configuration is done per collection. + + Following are optional parameters for MongoDB backend configuration, + + :param username: string, the name of the user to authenticate. + + :param password: string, the password of the user to authenticate. + + :param max_pool_size: integer, the maximum number of connections that the + pool will open simultaneously. By default the pool size is 10. + + :param w: integer, write acknowledgement for MongoDB client + + If not provided, then no default is set on MongoDB and then write + acknowledgement behavior occurs as per MongoDB default. This parameter + name is same as what is used in MongoDB docs. This value is specified + at collection level so its applicable to `cache_collection` db write + operations. + + If this is a replica set, write operations will block until they have + been replicated to the specified number or tagged set of servers. + Setting w=0 disables write acknowledgement and all other write concern + options. + + :param read_preference: string, the read preference mode for MongoDB client + Expected value is ``primary``, ``primaryPreferred``, ``secondary``, + ``secondaryPreferred``, or ``nearest``. This read_preference is + specified at collection level so its applicable to `cache_collection` + db read operations. + + :param use_replica: boolean, flag to indicate if replica client to be + used. Default is `False`. `replicaset_name` value is required if + `True`. + + :param replicaset_name: string, name of replica set. + Becomes required if `use_replica` is `True` + + :param son_manipulator: string, name of class with module name which + implements MongoDB SONManipulator. + Default manipulator used is :class:`.BaseTransform`. + + This manipulator is added per database. In multiple cache + configurations, the manipulator name should be same if same + database name ``db_name`` is used in those configurations. + + SONManipulator is used to manipulate custom data types as they are + saved or retrieved from MongoDB. Custom impl is only needed if cached + data is custom class and needs transformations when saving or reading + from db. If dogpile cached value contains built-in data types, then + BaseTransform class is sufficient as it already handles dogpile + CachedValue class transformation. + + :param mongo_ttl_seconds: integer, interval in seconds to indicate maximum + time-to-live value. + If value is greater than 0, then its assumed that cache_collection + needs to be TTL type (has index at 'doc_date' field). + By default, the value is -1 and its disabled. + Reference: <http://docs.mongodb.org/manual/tutorial/expire-data/> + + .. NOTE:: + + This parameter is different from Dogpile own + expiration_time, which is the number of seconds after which Dogpile + will consider the value to be expired. When Dogpile considers a + value to be expired, it continues to use the value until generation + of a new value is complete, when using CacheRegion.get_or_create(). + Therefore, if you are setting `mongo_ttl_seconds`, you will want to + make sure it is greater than expiration_time by at least enough + seconds for new values to be generated, else the value would not + be available during a regeneration, forcing all threads to wait for + a regeneration each time a value expires. + + :param ssl: boolean, If True, create the connection to the server + using SSL. Default is `False`. Client SSL connection parameters depends + on server side SSL setup. For further reference on SSL configuration: + <http://docs.mongodb.org/manual/tutorial/configure-ssl/> + + :param ssl_keyfile: string, the private keyfile used to identify the + local connection against mongod. If included with the certfile then + only the `ssl_certfile` is needed. Used only when `ssl` is `True`. + + :param ssl_certfile: string, the certificate file used to identify the + local connection against mongod. Used only when `ssl` is `True`. + + :param ssl_ca_certs: string, the ca_certs file contains a set of + concatenated 'certification authority' certificates, which are used to + validate certificates passed from the other end of the connection. + Used only when `ssl` is `True`. + + :param ssl_cert_reqs: string, the parameter cert_reqs specifies whether + a certificate is required from the other side of the connection, and + whether it will be validated if provided. It must be one of the three + values ``ssl.CERT_NONE`` (certificates ignored), ``ssl.CERT_OPTIONAL`` + (not required, but validated if provided), or + ``ssl.CERT_REQUIRED`` (required and validated). If the value of this + parameter is not ``ssl.CERT_NONE``, then the ssl_ca_certs parameter + must point to a file of CA certificates. Used only when `ssl` + is `True`. + + Rest of arguments are passed to mongo calls for read, write and remove. + So related options can be specified to pass to these operations. + + Further details of various supported arguments can be referred from + <http://api.mongodb.org/python/current/api/pymongo/> + + """ + + def __init__(self, arguments): + self.api = MongoApi(arguments) + + @dp_util.memoized_property + def client(self): + """Initializes MongoDB connection and collection defaults. + + This initialization is done only once and performed as part of lazy + inclusion of MongoDB dependency i.e. add imports only if related + backend is used. + + :return: :class:`.MongoApi` instance + """ + self.api.get_cache_collection() + return self.api + + def get(self, key): + value = self.client.get(key) + if value is None: + return NO_VALUE + else: + return value + + def get_multi(self, keys): + values = self.client.get_multi(keys) + return [ + NO_VALUE if key not in values + else values[key] for key in keys + ] + + def set(self, key, value): + self.client.set(key, value) + + def set_multi(self, mapping): + self.client.set_multi(mapping) + + def delete(self, key): + self.client.delete(key) + + def delete_multi(self, keys): + self.client.delete_multi(keys) + + +class MongoApi(object): + """Class handling MongoDB specific functionality. + + This class uses PyMongo APIs internally to create database connection + with configured pool size, ensures unique index on key, does database + authentication and ensure TTL collection index if configured so. + This class also serves as handle to cache collection for dogpile cache + APIs. + + In a single deployment, multiple cache configuration can be defined. In + that case of multiple cache collections usage, db client connection pool + is shared when cache collections are within same database. + """ + + # class level attributes for re-use of db client connection and collection + _DB = {} # dict of db_name: db connection reference + _MONGO_COLLS = {} # dict of cache_collection : db collection reference + + def __init__(self, arguments): + self._init_args(arguments) + self._data_manipulator = None + + def _init_args(self, arguments): + """Helper logic for collecting and parsing MongoDB specific arguments. + + The arguments passed in are separated out in connection specific + setting and rest of arguments are passed to create/update/delete + db operations. + """ + self.conn_kwargs = {} # connection specific arguments + + self.hosts = arguments.pop('db_hosts', None) + if self.hosts is None: + msg = _('db_hosts value is required') + raise exception.ValidationError(message=msg) + + self.db_name = arguments.pop('db_name', None) + if self.db_name is None: + msg = _('database db_name is required') + raise exception.ValidationError(message=msg) + + self.cache_collection = arguments.pop('cache_collection', None) + if self.cache_collection is None: + msg = _('cache_collection name is required') + raise exception.ValidationError(message=msg) + + self.username = arguments.pop('username', None) + self.password = arguments.pop('password', None) + self.max_pool_size = arguments.pop('max_pool_size', 10) + + self.w = arguments.pop('w', -1) + try: + self.w = int(self.w) + except ValueError: + msg = _('integer value expected for w (write concern attribute)') + raise exception.ValidationError(message=msg) + + self.read_preference = arguments.pop('read_preference', None) + + self.use_replica = arguments.pop('use_replica', False) + if self.use_replica: + if arguments.get('replicaset_name') is None: + msg = _('replicaset_name required when use_replica is True') + raise exception.ValidationError(message=msg) + self.replicaset_name = arguments.get('replicaset_name') + + self.son_manipulator = arguments.pop('son_manipulator', None) + + # set if mongo collection needs to be TTL type. + # This needs to be max ttl for any cache entry. + # By default, -1 means don't use TTL collection. + # With ttl set, it creates related index and have doc_date field with + # needed expiration interval + self.ttl_seconds = arguments.pop('mongo_ttl_seconds', -1) + try: + self.ttl_seconds = int(self.ttl_seconds) + except ValueError: + msg = _('integer value expected for mongo_ttl_seconds') + raise exception.ValidationError(message=msg) + + self.conn_kwargs['ssl'] = arguments.pop('ssl', False) + if self.conn_kwargs['ssl']: + ssl_keyfile = arguments.pop('ssl_keyfile', None) + ssl_certfile = arguments.pop('ssl_certfile', None) + ssl_ca_certs = arguments.pop('ssl_ca_certs', None) + ssl_cert_reqs = arguments.pop('ssl_cert_reqs', None) + if ssl_keyfile: + self.conn_kwargs['ssl_keyfile'] = ssl_keyfile + if ssl_certfile: + self.conn_kwargs['ssl_certfile'] = ssl_certfile + if ssl_ca_certs: + self.conn_kwargs['ssl_ca_certs'] = ssl_ca_certs + if ssl_cert_reqs: + self.conn_kwargs['ssl_cert_reqs'] = ( + self._ssl_cert_req_type(ssl_cert_reqs)) + + # rest of arguments are passed to mongo crud calls + self.meth_kwargs = arguments + + def _ssl_cert_req_type(self, req_type): + try: + import ssl + except ImportError: + raise exception.ValidationError(_('no ssl support available')) + req_type = req_type.upper() + try: + return { + 'NONE': ssl.CERT_NONE, + 'OPTIONAL': ssl.CERT_OPTIONAL, + 'REQUIRED': ssl.CERT_REQUIRED + }[req_type] + except KeyError: + msg = _('Invalid ssl_cert_reqs value of %s, must be one of ' + '"NONE", "OPTIONAL", "REQUIRED"') % (req_type) + raise exception.ValidationError(message=msg) + + def _get_db(self): + # defer imports until backend is used + global pymongo + import pymongo + if self.use_replica: + connection = pymongo.MongoReplicaSetClient( + host=self.hosts, replicaSet=self.replicaset_name, + max_pool_size=self.max_pool_size, **self.conn_kwargs) + else: # used for standalone node or mongos in sharded setup + connection = pymongo.MongoClient( + host=self.hosts, max_pool_size=self.max_pool_size, + **self.conn_kwargs) + + database = getattr(connection, self.db_name) + + self._assign_data_mainpulator() + database.add_son_manipulator(self._data_manipulator) + if self.username and self.password: + database.authenticate(self.username, self.password) + return database + + def _assign_data_mainpulator(self): + if self._data_manipulator is None: + if self.son_manipulator: + self._data_manipulator = importutils.import_object( + self.son_manipulator) + else: + self._data_manipulator = BaseTransform() + + def _get_doc_date(self): + if self.ttl_seconds > 0: + expire_delta = datetime.timedelta(seconds=self.ttl_seconds) + doc_date = timeutils.utcnow() + expire_delta + else: + doc_date = timeutils.utcnow() + return doc_date + + def get_cache_collection(self): + if self.cache_collection not in self._MONGO_COLLS: + global pymongo + import pymongo + # re-use db client connection if already defined as part of + # earlier dogpile cache configuration + if self.db_name not in self._DB: + self._DB[self.db_name] = self._get_db() + coll = getattr(self._DB[self.db_name], self.cache_collection) + + self._assign_data_mainpulator() + if self.read_preference: + self.read_preference = pymongo.read_preferences.mongos_enum( + self.read_preference) + coll.read_preference = self.read_preference + if self.w > -1: + coll.write_concern['w'] = self.w + if self.ttl_seconds > 0: + kwargs = {'expireAfterSeconds': self.ttl_seconds} + coll.ensure_index('doc_date', cache_for=5, **kwargs) + else: + self._validate_ttl_index(coll, self.cache_collection, + self.ttl_seconds) + self._MONGO_COLLS[self.cache_collection] = coll + + return self._MONGO_COLLS[self.cache_collection] + + def _get_cache_entry(self, key, value, meta, doc_date): + """MongoDB cache data representation. + + Storing cache key as ``_id`` field as MongoDB by default creates + unique index on this field. So no need to create separate field and + index for storing cache key. Cache data has additional ``doc_date`` + field for MongoDB TTL collection support. + """ + return dict(_id=key, value=value, meta=meta, doc_date=doc_date) + + def _validate_ttl_index(self, collection, coll_name, ttl_seconds): + """Checks if existing TTL index is removed on a collection. + + This logs warning when existing collection has TTL index defined and + new cache configuration tries to disable index with + ``mongo_ttl_seconds < 0``. In that case, existing index needs + to be addressed first to make new configuration effective. + Refer to MongoDB documentation around TTL index for further details. + """ + indexes = collection.index_information() + for indx_name, index_data in six.iteritems(indexes): + if all(k in index_data for k in ('key', 'expireAfterSeconds')): + existing_value = index_data['expireAfterSeconds'] + fld_present = 'doc_date' in index_data['key'][0] + if fld_present and existing_value > -1 and ttl_seconds < 1: + msg = _LW('TTL index already exists on db collection ' + '<%(c_name)s>, remove index <%(indx_name)s> ' + 'first to make updated mongo_ttl_seconds value ' + 'to be effective') + LOG.warn(msg, {'c_name': coll_name, + 'indx_name': indx_name}) + + def get(self, key): + critieria = {'_id': key} + result = self.get_cache_collection().find_one(spec_or_id=critieria, + **self.meth_kwargs) + if result: + return result['value'] + else: + return None + + def get_multi(self, keys): + db_results = self._get_results_as_dict(keys) + return {doc['_id']: doc['value'] for doc in six.itervalues(db_results)} + + def _get_results_as_dict(self, keys): + critieria = {'_id': {'$in': keys}} + db_results = self.get_cache_collection().find(spec=critieria, + **self.meth_kwargs) + return {doc['_id']: doc for doc in db_results} + + def set(self, key, value): + doc_date = self._get_doc_date() + ref = self._get_cache_entry(key, value.payload, value.metadata, + doc_date) + spec = {'_id': key} + # find and modify does not have manipulator support + # so need to do conversion as part of input document + ref = self._data_manipulator.transform_incoming(ref, self) + self.get_cache_collection().find_and_modify(spec, ref, upsert=True, + **self.meth_kwargs) + + def set_multi(self, mapping): + """Insert multiple documents specified as key, value pairs. + + In this case, multiple documents can be added via insert provided they + do not exist. + Update of multiple existing documents is done one by one + """ + doc_date = self._get_doc_date() + insert_refs = [] + update_refs = [] + existing_docs = self._get_results_as_dict(mapping.keys()) + for key, value in mapping.items(): + ref = self._get_cache_entry(key, value.payload, value.metadata, + doc_date) + if key in existing_docs: + ref['_id'] = existing_docs[key]['_id'] + update_refs.append(ref) + else: + insert_refs.append(ref) + if insert_refs: + self.get_cache_collection().insert(insert_refs, manipulate=True, + **self.meth_kwargs) + for upd_doc in update_refs: + self.get_cache_collection().save(upd_doc, manipulate=True, + **self.meth_kwargs) + + def delete(self, key): + critieria = {'_id': key} + self.get_cache_collection().remove(spec_or_id=critieria, + **self.meth_kwargs) + + def delete_multi(self, keys): + critieria = {'_id': {'$in': keys}} + self.get_cache_collection().remove(spec_or_id=critieria, + **self.meth_kwargs) + + +@six.add_metaclass(abc.ABCMeta) +class AbstractManipulator(object): + """Abstract class with methods which need to be implemented for custom + manipulation. + + Adding this as a base class for :class:`.BaseTransform` instead of adding + import dependency of pymongo specific class i.e. + `pymongo.son_manipulator.SONManipulator` and using that as base class. + This is done to avoid pymongo dependency if MongoDB backend is not used. + """ + @abc.abstractmethod + def transform_incoming(self, son, collection): + """Used while saving data to MongoDB. + + :param son: the SON object to be inserted into the database + :param collection: the collection the object is being inserted into + + :returns: transformed SON object + + """ + raise exception.NotImplemented() # pragma: no cover + + @abc.abstractmethod + def transform_outgoing(self, son, collection): + """Used while reading data from MongoDB. + + :param son: the SON object being retrieved from the database + :param collection: the collection this object was stored in + + :returns: transformed SON object + """ + raise exception.NotImplemented() # pragma: no cover + + def will_copy(self): + """Will this SON manipulator make a copy of the incoming document? + + Derived classes that do need to make a copy should override this + method, returning `True` instead of `False`. + + :returns: boolean + """ + return False + + +class BaseTransform(AbstractManipulator): + """Base transformation class to store and read dogpile cached data + from MongoDB. + + This is needed as dogpile internally stores data as a custom class + i.e. dogpile.cache.api.CachedValue + + Note: Custom manipulator needs to always override ``transform_incoming`` + and ``transform_outgoing`` methods. MongoDB manipulator logic specifically + checks that overridden method in instance and its super are different. + """ + + def transform_incoming(self, son, collection): + """Used while saving data to MongoDB.""" + for (key, value) in son.items(): + if isinstance(value, api.CachedValue): + son[key] = value.payload # key is 'value' field here + son['meta'] = value.metadata + elif isinstance(value, dict): # Make sure we recurse into sub-docs + son[key] = self.transform_incoming(value, collection) + return son + + def transform_outgoing(self, son, collection): + """Used while reading data from MongoDB.""" + metadata = None + # make sure its top level dictionary with all expected fields names + # present + if isinstance(son, dict) and all(k in son for k in + ('_id', 'value', 'meta', 'doc_date')): + payload = son.pop('value', None) + metadata = son.pop('meta', None) + for (key, value) in son.items(): + if isinstance(value, dict): + son[key] = self.transform_outgoing(value, collection) + if metadata is not None: + son['value'] = api.CachedValue(payload, metadata) + return son diff --git a/keystone-moon/keystone/common/cache/backends/noop.py b/keystone-moon/keystone/common/cache/backends/noop.py new file mode 100644 index 00000000..38329c94 --- /dev/null +++ b/keystone-moon/keystone/common/cache/backends/noop.py @@ -0,0 +1,49 @@ +# Copyright 2013 Metacloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from dogpile.cache import api + + +NO_VALUE = api.NO_VALUE + + +class NoopCacheBackend(api.CacheBackend): + """A no op backend as a default caching backend. + + The no op backend is provided as the default caching backend for keystone + to ensure that ``dogpile.cache.memory`` is not used in any real-world + circumstances unintentionally. ``dogpile.cache.memory`` does not have a + mechanism to cleanup it's internal dict and therefore could cause run-away + memory utilization. + """ + def __init__(self, *args): + return + + def get(self, key): + return NO_VALUE + + def get_multi(self, keys): + return [NO_VALUE for x in keys] + + def set(self, key, value): + return + + def set_multi(self, mapping): + return + + def delete(self, key): + return + + def delete_multi(self, keys): + return diff --git a/keystone-moon/keystone/common/cache/core.py b/keystone-moon/keystone/common/cache/core.py new file mode 100644 index 00000000..306587b3 --- /dev/null +++ b/keystone-moon/keystone/common/cache/core.py @@ -0,0 +1,308 @@ +# Copyright 2013 Metacloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Keystone Caching Layer Implementation.""" + +import dogpile.cache +from dogpile.cache import proxy +from dogpile.cache import util +from oslo_config import cfg +from oslo_log import log +from oslo_utils import importutils + +from keystone import exception +from keystone.i18n import _, _LE + + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + +make_region = dogpile.cache.make_region + +dogpile.cache.register_backend( + 'keystone.common.cache.noop', + 'keystone.common.cache.backends.noop', + 'NoopCacheBackend') + +dogpile.cache.register_backend( + 'keystone.cache.mongo', + 'keystone.common.cache.backends.mongo', + 'MongoCacheBackend') + +dogpile.cache.register_backend( + 'keystone.cache.memcache_pool', + 'keystone.common.cache.backends.memcache_pool', + 'PooledMemcachedBackend') + + +class DebugProxy(proxy.ProxyBackend): + """Extra Logging ProxyBackend.""" + # NOTE(morganfainberg): Pass all key/values through repr to ensure we have + # a clean description of the information. Without use of repr, it might + # be possible to run into encode/decode error(s). For logging/debugging + # purposes encode/decode is irrelevant and we should be looking at the + # data exactly as it stands. + + def get(self, key): + value = self.proxied.get(key) + LOG.debug('CACHE_GET: Key: "%(key)r" Value: "%(value)r"', + {'key': key, 'value': value}) + return value + + def get_multi(self, keys): + values = self.proxied.get_multi(keys) + LOG.debug('CACHE_GET_MULTI: "%(keys)r" Values: "%(values)r"', + {'keys': keys, 'values': values}) + return values + + def set(self, key, value): + LOG.debug('CACHE_SET: Key: "%(key)r" Value: "%(value)r"', + {'key': key, 'value': value}) + return self.proxied.set(key, value) + + def set_multi(self, keys): + LOG.debug('CACHE_SET_MULTI: "%r"', keys) + self.proxied.set_multi(keys) + + def delete(self, key): + self.proxied.delete(key) + LOG.debug('CACHE_DELETE: "%r"', key) + + def delete_multi(self, keys): + LOG.debug('CACHE_DELETE_MULTI: "%r"', keys) + self.proxied.delete_multi(keys) + + +def build_cache_config(): + """Build the cache region dictionary configuration. + + :returns: dict + """ + prefix = CONF.cache.config_prefix + conf_dict = {} + conf_dict['%s.backend' % prefix] = CONF.cache.backend + conf_dict['%s.expiration_time' % prefix] = CONF.cache.expiration_time + for argument in CONF.cache.backend_argument: + try: + (argname, argvalue) = argument.split(':', 1) + except ValueError: + msg = _LE('Unable to build cache config-key. Expected format ' + '"<argname>:<value>". Skipping unknown format: %s') + LOG.error(msg, argument) + continue + + arg_key = '.'.join([prefix, 'arguments', argname]) + conf_dict[arg_key] = argvalue + + LOG.debug('Keystone Cache Config: %s', conf_dict) + # NOTE(yorik-sar): these arguments will be used for memcache-related + # backends. Use setdefault for url to support old-style setting through + # backend_argument=url:127.0.0.1:11211 + conf_dict.setdefault('%s.arguments.url' % prefix, + CONF.cache.memcache_servers) + for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize', + 'pool_unused_timeout', 'pool_connection_get_timeout'): + value = getattr(CONF.cache, 'memcache_' + arg) + conf_dict['%s.arguments.%s' % (prefix, arg)] = value + + return conf_dict + + +def configure_cache_region(region): + """Configure a cache region. + + :param region: optional CacheRegion object, if not provided a new region + will be instantiated + :raises: exception.ValidationError + :returns: dogpile.cache.CacheRegion + """ + if not isinstance(region, dogpile.cache.CacheRegion): + raise exception.ValidationError( + _('region not type dogpile.cache.CacheRegion')) + + if not region.is_configured: + # NOTE(morganfainberg): this is how you tell if a region is configured. + # There is a request logged with dogpile.cache upstream to make this + # easier / less ugly. + + config_dict = build_cache_config() + region.configure_from_config(config_dict, + '%s.' % CONF.cache.config_prefix) + + if CONF.cache.debug_cache_backend: + region.wrap(DebugProxy) + + # NOTE(morganfainberg): if the backend requests the use of a + # key_mangler, we should respect that key_mangler function. If a + # key_mangler is not defined by the backend, use the sha1_mangle_key + # mangler provided by dogpile.cache. This ensures we always use a fixed + # size cache-key. + if region.key_mangler is None: + region.key_mangler = util.sha1_mangle_key + + for class_path in CONF.cache.proxies: + # NOTE(morganfainberg): if we have any proxy wrappers, we should + # ensure they are added to the cache region's backend. Since + # configure_from_config doesn't handle the wrap argument, we need + # to manually add the Proxies. For information on how the + # ProxyBackends work, see the dogpile.cache documents on + # "changing-backend-behavior" + cls = importutils.import_class(class_path) + LOG.debug("Adding cache-proxy '%s' to backend.", class_path) + region.wrap(cls) + + return region + + +def get_should_cache_fn(section): + """Build a function that returns a config section's caching status. + + For any given driver in keystone that has caching capabilities, a boolean + config option for that driver's section (e.g. ``token``) should exist and + default to ``True``. This function will use that value to tell the caching + decorator if caching for that driver is enabled. To properly use this + with the decorator, pass this function the configuration section and assign + the result to a variable. Pass the new variable to the caching decorator + as the named argument ``should_cache_fn``. e.g.:: + + from keystone.common import cache + + SHOULD_CACHE = cache.get_should_cache_fn('token') + + @cache.on_arguments(should_cache_fn=SHOULD_CACHE) + def function(arg1, arg2): + ... + + :param section: name of the configuration section to examine + :type section: string + :returns: function reference + """ + def should_cache(value): + if not CONF.cache.enabled: + return False + conf_group = getattr(CONF, section) + return getattr(conf_group, 'caching', True) + return should_cache + + +def get_expiration_time_fn(section): + """Build a function that returns a config section's expiration time status. + + For any given driver in keystone that has caching capabilities, an int + config option called ``cache_time`` for that driver's section + (e.g. ``token``) should exist and typically default to ``None``. This + function will use that value to tell the caching decorator of the TTL + override for caching the resulting objects. If the value of the config + option is ``None`` the default value provided in the + ``[cache] expiration_time`` option will be used by the decorator. The + default may be set to something other than ``None`` in cases where the + caching TTL should not be tied to the global default(s) (e.g. + revocation_list changes very infrequently and can be cached for >1h by + default). + + To properly use this with the decorator, pass this function the + configuration section and assign the result to a variable. Pass the new + variable to the caching decorator as the named argument + ``expiration_time``. e.g.:: + + from keystone.common import cache + + EXPIRATION_TIME = cache.get_expiration_time_fn('token') + + @cache.on_arguments(expiration_time=EXPIRATION_TIME) + def function(arg1, arg2): + ... + + :param section: name of the configuration section to examine + :type section: string + :rtype: function reference + """ + def get_expiration_time(): + conf_group = getattr(CONF, section) + return getattr(conf_group, 'cache_time', None) + return get_expiration_time + + +def key_generate_to_str(s): + # NOTE(morganfainberg): Since we need to stringify all arguments, attempt + # to stringify and handle the Unicode error explicitly as needed. + try: + return str(s) + except UnicodeEncodeError: + return s.encode('utf-8') + + +def function_key_generator(namespace, fn, to_str=key_generate_to_str): + # NOTE(morganfainberg): This wraps dogpile.cache's default + # function_key_generator to change the default to_str mechanism. + return util.function_key_generator(namespace, fn, to_str=to_str) + + +REGION = dogpile.cache.make_region( + function_key_generator=function_key_generator) +on_arguments = REGION.cache_on_arguments + + +def get_memoization_decorator(section, expiration_section=None): + """Build a function based on the `on_arguments` decorator for the section. + + For any given driver in Keystone that has caching capabilities, a + pair of functions is required to properly determine the status of the + caching capabilities (a toggle to indicate caching is enabled and any + override of the default TTL for cached data). This function will return + an object that has the memoization decorator ``on_arguments`` + pre-configured for the driver. + + Example usage:: + + from keystone.common import cache + + MEMOIZE = cache.get_memoization_decorator(section='token') + + @MEMOIZE + def function(arg1, arg2): + ... + + + ALTERNATE_MEMOIZE = cache.get_memoization_decorator( + section='token', expiration_section='revoke') + + @ALTERNATE_MEMOIZE + def function2(arg1, arg2): + ... + + :param section: name of the configuration section to examine + :type section: string + :param expiration_section: name of the configuration section to examine + for the expiration option. This will fall back + to using ``section`` if the value is unspecified + or ``None`` + :type expiration_section: string + :rtype: function reference + """ + if expiration_section is None: + expiration_section = section + should_cache = get_should_cache_fn(section) + expiration_time = get_expiration_time_fn(expiration_section) + + memoize = REGION.cache_on_arguments(should_cache_fn=should_cache, + expiration_time=expiration_time) + + # Make sure the actual "should_cache" and "expiration_time" methods are + # available. This is potentially interesting/useful to pre-seed cache + # values. + memoize.should_cache = should_cache + memoize.get_expiration_time = expiration_time + + return memoize |