summaryrefslogtreecommitdiffstats
path: root/keystone-moon/keystone/common/kvs/backends
diff options
context:
space:
mode:
Diffstat (limited to 'keystone-moon/keystone/common/kvs/backends')
-rw-r--r--keystone-moon/keystone/common/kvs/backends/__init__.py0
-rw-r--r--keystone-moon/keystone/common/kvs/backends/inmemdb.py69
-rw-r--r--keystone-moon/keystone/common/kvs/backends/memcached.py188
3 files changed, 257 insertions, 0 deletions
diff --git a/keystone-moon/keystone/common/kvs/backends/__init__.py b/keystone-moon/keystone/common/kvs/backends/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone-moon/keystone/common/kvs/backends/__init__.py
diff --git a/keystone-moon/keystone/common/kvs/backends/inmemdb.py b/keystone-moon/keystone/common/kvs/backends/inmemdb.py
new file mode 100644
index 00000000..68072ef4
--- /dev/null
+++ b/keystone-moon/keystone/common/kvs/backends/inmemdb.py
@@ -0,0 +1,69 @@
+# Copyright 2013 Metacloud, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Keystone In-Memory Dogpile.cache backend implementation.
+"""
+
+import copy
+
+from dogpile.cache import api
+
+
+NO_VALUE = api.NO_VALUE
+
+
+class MemoryBackend(api.CacheBackend):
+ """A backend that uses a plain dictionary.
+
+ There is no size management, and values which are placed into the
+ dictionary will remain until explicitly removed. Note that Dogpile's
+ expiration of items is based on timestamps and does not remove them from
+ the cache.
+
+ E.g.::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'keystone.common.kvs.Memory'
+ )
+ """
+ def __init__(self, arguments):
+ self._db = {}
+
+ def _isolate_value(self, value):
+ if value is not NO_VALUE:
+ return copy.deepcopy(value)
+ return value
+
+ def get(self, key):
+ return self._isolate_value(self._db.get(key, NO_VALUE))
+
+ def get_multi(self, keys):
+ return [self.get(key) for key in keys]
+
+ def set(self, key, value):
+ self._db[key] = self._isolate_value(value)
+
+ def set_multi(self, mapping):
+ for key, value in mapping.items():
+ self.set(key, value)
+
+ def delete(self, key):
+ self._db.pop(key, None)
+
+ def delete_multi(self, keys):
+ for key in keys:
+ self.delete(key)
diff --git a/keystone-moon/keystone/common/kvs/backends/memcached.py b/keystone-moon/keystone/common/kvs/backends/memcached.py
new file mode 100644
index 00000000..db453143
--- /dev/null
+++ b/keystone-moon/keystone/common/kvs/backends/memcached.py
@@ -0,0 +1,188 @@
+# Copyright 2013 Metacloud, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Keystone Memcached dogpile.cache backend implementation.
+"""
+
+import random as _random
+import time
+
+from dogpile.cache import api
+from dogpile.cache.backends import memcached
+from oslo_config import cfg
+from oslo_log import log
+
+from keystone.common.cache.backends import memcache_pool
+from keystone.common import manager
+from keystone import exception
+from keystone.i18n import _
+
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+NO_VALUE = api.NO_VALUE
+random = _random.SystemRandom()
+
+VALID_DOGPILE_BACKENDS = dict(
+ pylibmc=memcached.PylibmcBackend,
+ bmemcached=memcached.BMemcachedBackend,
+ memcached=memcached.MemcachedBackend,
+ pooled_memcached=memcache_pool.PooledMemcachedBackend)
+
+
+class MemcachedLock(object):
+ """Simple distributed lock using memcached.
+
+ This is an adaptation of the lock featured at
+ http://amix.dk/blog/post/19386
+
+ """
+ def __init__(self, client_fn, key, lock_timeout, max_lock_attempts):
+ self.client_fn = client_fn
+ self.key = "_lock" + key
+ self.lock_timeout = lock_timeout
+ self.max_lock_attempts = max_lock_attempts
+
+ def acquire(self, wait=True):
+ client = self.client_fn()
+ for i in range(self.max_lock_attempts):
+ if client.add(self.key, 1, self.lock_timeout):
+ return True
+ elif not wait:
+ return False
+ else:
+ sleep_time = random.random()
+ time.sleep(sleep_time)
+ raise exception.UnexpectedError(
+ _('Maximum lock attempts on %s occurred.') % self.key)
+
+ def release(self):
+ client = self.client_fn()
+ client.delete(self.key)
+
+
+class MemcachedBackend(manager.Manager):
+ """Pivot point to leverage the various dogpile.cache memcached backends.
+
+ To specify a specific dogpile.cache memcached driver, pass the argument
+ `memcached_driver` set to one of the provided memcached drivers (at this
+ time `memcached`, `bmemcached`, `pylibmc` are valid).
+ """
+ def __init__(self, arguments):
+ self._key_mangler = None
+ self.raw_no_expiry_keys = set(arguments.pop('no_expiry_keys', set()))
+ self.no_expiry_hashed_keys = set()
+
+ self.lock_timeout = arguments.pop('lock_timeout', None)
+ self.max_lock_attempts = arguments.pop('max_lock_attempts', 15)
+ # NOTE(morganfainberg): Remove distributed locking from the arguments
+ # passed to the "real" backend if it exists.
+ arguments.pop('distributed_lock', None)
+ backend = arguments.pop('memcached_backend', None)
+ if 'url' not in arguments:
+ # FIXME(morganfainberg): Log deprecation warning for old-style
+ # configuration once full dict_config style configuration for
+ # KVS backends is supported. For now use the current memcache
+ # section of the configuration.
+ arguments['url'] = CONF.memcache.servers
+
+ if backend is None:
+ # NOTE(morganfainberg): Use the basic memcached backend if nothing
+ # else is supplied.
+ self.driver = VALID_DOGPILE_BACKENDS['memcached'](arguments)
+ else:
+ if backend not in VALID_DOGPILE_BACKENDS:
+ raise ValueError(
+ _('Backend `%(driver)s` is not a valid memcached '
+ 'backend. Valid drivers: %(driver_list)s') %
+ {'driver': backend,
+ 'driver_list': ','.join(VALID_DOGPILE_BACKENDS.keys())})
+ else:
+ self.driver = VALID_DOGPILE_BACKENDS[backend](arguments)
+
+ def _get_set_arguments_driver_attr(self, exclude_expiry=False):
+
+ # NOTE(morganfainberg): Shallow copy the .set_arguments dict to
+ # ensure no changes cause the values to change in the instance
+ # variable.
+ set_arguments = getattr(self.driver, 'set_arguments', {}).copy()
+
+ if exclude_expiry:
+ # NOTE(morganfainberg): Explicitly strip out the 'time' key/value
+ # from the set_arguments in the case that this key isn't meant
+ # to expire
+ set_arguments.pop('time', None)
+ return set_arguments
+
+ def set(self, key, value):
+ mapping = {key: value}
+ self.set_multi(mapping)
+
+ def set_multi(self, mapping):
+ mapping_keys = set(mapping.keys())
+ no_expiry_keys = mapping_keys.intersection(self.no_expiry_hashed_keys)
+ has_expiry_keys = mapping_keys.difference(self.no_expiry_hashed_keys)
+
+ if no_expiry_keys:
+ # NOTE(morganfainberg): For keys that have expiry excluded,
+ # bypass the backend and directly call the client. Bypass directly
+ # to the client is required as the 'set_arguments' are applied to
+ # all ``set`` and ``set_multi`` calls by the driver, by calling
+ # the client directly it is possible to exclude the ``time``
+ # argument to the memcached server.
+ new_mapping = {k: mapping[k] for k in no_expiry_keys}
+ set_arguments = self._get_set_arguments_driver_attr(
+ exclude_expiry=True)
+ self.driver.client.set_multi(new_mapping, **set_arguments)
+
+ if has_expiry_keys:
+ new_mapping = {k: mapping[k] for k in has_expiry_keys}
+ self.driver.set_multi(new_mapping)
+
+ @classmethod
+ def from_config_dict(cls, config_dict, prefix):
+ prefix_len = len(prefix)
+ return cls(
+ {key[prefix_len:]: config_dict[key] for key in config_dict
+ if key.startswith(prefix)})
+
+ @property
+ def key_mangler(self):
+ if self._key_mangler is None:
+ self._key_mangler = self.driver.key_mangler
+ return self._key_mangler
+
+ @key_mangler.setter
+ def key_mangler(self, key_mangler):
+ if callable(key_mangler):
+ self._key_mangler = key_mangler
+ self._rehash_keys()
+ elif key_mangler is None:
+ # NOTE(morganfainberg): Set the hashed key map to the unhashed
+ # list since we no longer have a key_mangler.
+ self._key_mangler = None
+ self.no_expiry_hashed_keys = self.raw_no_expiry_keys
+ else:
+ raise TypeError(_('`key_mangler` functions must be callable.'))
+
+ def _rehash_keys(self):
+ no_expire = set()
+ for key in self.raw_no_expiry_keys:
+ no_expire.add(self._key_mangler(key))
+ self.no_expiry_hashed_keys = no_expire
+
+ def get_mutex(self, key):
+ return MemcachedLock(lambda: self.driver.client, key,
+ self.lock_timeout, self.max_lock_attempts)