diff options
Diffstat (limited to 'keystone-moon/keystone/tests/unit/test_cache_backend_mongo.py')
| -rw-r--r-- | keystone-moon/keystone/tests/unit/test_cache_backend_mongo.py | 727 |
1 files changed, 727 insertions, 0 deletions
diff --git a/keystone-moon/keystone/tests/unit/test_cache_backend_mongo.py b/keystone-moon/keystone/tests/unit/test_cache_backend_mongo.py new file mode 100644 index 00000000..a56bf754 --- /dev/null +++ b/keystone-moon/keystone/tests/unit/test_cache_backend_mongo.py @@ -0,0 +1,727 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import copy +import functools +import uuid + +from dogpile.cache import api +from dogpile.cache import region as dp_region +import six + +from keystone.common.cache.backends import mongo +from keystone import exception +from keystone.tests import unit as tests + + +# Mock database structure sample where 'ks_cache' is database and +# 'cache' is collection. Dogpile CachedValue data is divided in two +# fields `value` (CachedValue.payload) and `meta` (CachedValue.metadata) +ks_cache = { + "cache": [ + { + "value": { + "serviceType": "identity", + "allVersionsUrl": "https://dummyUrl", + "dateLastModified": "ISODDate(2014-02-08T18:39:13.237Z)", + "serviceName": "Identity", + "enabled": "True" + }, + "meta": { + "v": 1, + "ct": 1392371422.015121 + }, + "doc_date": "ISODate('2014-02-14T09:50:22.015Z')", + "_id": "8251dc95f63842719c077072f1047ddf" + }, + { + "value": "dummyValueX", + "meta": { + "v": 1, + "ct": 1392371422.014058 + }, + "doc_date": "ISODate('2014-02-14T09:50:22.014Z')", + "_id": "66730b9534d146f0804d23729ad35436" + } + ] +} + + +COLLECTIONS = {} +SON_MANIPULATOR = None + + +class MockCursor(object): + + def __init__(self, collection, dataset_factory): + super(MockCursor, self).__init__() + self.collection = collection + self._factory = dataset_factory + self._dataset = self._factory() + self._limit = None + self._skip = None + + def __iter__(self): + return self + + def __next__(self): + if self._skip: + for _ in range(self._skip): + next(self._dataset) + self._skip = None + if self._limit is not None and self._limit <= 0: + raise StopIteration() + if self._limit is not None: + self._limit -= 1 + return next(self._dataset) + + next = __next__ + + def __getitem__(self, index): + arr = [x for x in self._dataset] + self._dataset = iter(arr) + return arr[index] + + +class MockCollection(object): + + def __init__(self, db, name): + super(MockCollection, self).__init__() + self.name = name + self._collection_database = db + self._documents = {} + self.write_concern = {} + + def __getattr__(self, name): + if name == 'database': + return self._collection_database + + def ensure_index(self, key_or_list, *args, **kwargs): + pass + + def index_information(self): + return {} + + def find_one(self, spec_or_id=None, *args, **kwargs): + if spec_or_id is None: + spec_or_id = {} + if not isinstance(spec_or_id, collections.Mapping): + spec_or_id = {'_id': spec_or_id} + + try: + return next(self.find(spec_or_id, *args, **kwargs)) + except StopIteration: + return None + + def find(self, spec=None, *args, **kwargs): + return MockCursor(self, functools.partial(self._get_dataset, spec)) + + def _get_dataset(self, spec): + dataset = (self._copy_doc(document, dict) for document in + self._iter_documents(spec)) + return dataset + + def _iter_documents(self, spec=None): + return (SON_MANIPULATOR.transform_outgoing(document, self) for + document in six.itervalues(self._documents) + if self._apply_filter(document, spec)) + + def _apply_filter(self, document, query): + for key, search in six.iteritems(query): + doc_val = document.get(key) + if isinstance(search, dict): + op_dict = {'$in': lambda dv, sv: dv in sv} + is_match = all( + op_str in op_dict and op_dict[op_str](doc_val, search_val) + for op_str, search_val in six.iteritems(search) + ) + else: + is_match = doc_val == search + + return is_match + + def _copy_doc(self, obj, container): + if isinstance(obj, list): + new = [] + for item in obj: + new.append(self._copy_doc(item, container)) + return new + if isinstance(obj, dict): + new = container() + for key, value in obj.items(): + new[key] = self._copy_doc(value, container) + return new + else: + return copy.copy(obj) + + def insert(self, data, manipulate=True, **kwargs): + if isinstance(data, list): + return [self._insert(element) for element in data] + return self._insert(data) + + def save(self, data, manipulate=True, **kwargs): + return self._insert(data) + + def _insert(self, data): + if '_id' not in data: + data['_id'] = uuid.uuid4().hex + object_id = data['_id'] + self._documents[object_id] = self._internalize_dict(data) + return object_id + + def find_and_modify(self, spec, document, upsert=False, **kwargs): + self.update(spec, document, upsert, **kwargs) + + def update(self, spec, document, upsert=False, **kwargs): + + existing_docs = [doc for doc in six.itervalues(self._documents) + if self._apply_filter(doc, spec)] + if existing_docs: + existing_doc = existing_docs[0] # should find only 1 match + _id = existing_doc['_id'] + existing_doc.clear() + existing_doc['_id'] = _id + existing_doc.update(self._internalize_dict(document)) + elif upsert: + existing_doc = self._documents[self._insert(document)] + + def _internalize_dict(self, d): + return {k: copy.deepcopy(v) for k, v in six.iteritems(d)} + + def remove(self, spec_or_id=None, search_filter=None): + """Remove objects matching spec_or_id from the collection.""" + if spec_or_id is None: + spec_or_id = search_filter if search_filter else {} + if not isinstance(spec_or_id, dict): + spec_or_id = {'_id': spec_or_id} + to_delete = list(self.find(spec=spec_or_id)) + for doc in to_delete: + doc_id = doc['_id'] + del self._documents[doc_id] + + return { + "connectionId": uuid.uuid4().hex, + "n": len(to_delete), + "ok": 1.0, + "err": None, + } + + +class MockMongoDB(object): + def __init__(self, dbname): + self._dbname = dbname + self.mainpulator = None + + def authenticate(self, username, password): + pass + + def add_son_manipulator(self, manipulator): + global SON_MANIPULATOR + SON_MANIPULATOR = manipulator + + def __getattr__(self, name): + if name == 'authenticate': + return self.authenticate + elif name == 'name': + return self._dbname + elif name == 'add_son_manipulator': + return self.add_son_manipulator + else: + return get_collection(self._dbname, name) + + def __getitem__(self, name): + return get_collection(self._dbname, name) + + +class MockMongoClient(object): + def __init__(self, *args, **kwargs): + pass + + def __getattr__(self, dbname): + return MockMongoDB(dbname) + + +def get_collection(db_name, collection_name): + mongo_collection = MockCollection(MockMongoDB(db_name), collection_name) + return mongo_collection + + +def pymongo_override(): + global pymongo + import pymongo + if pymongo.MongoClient is not MockMongoClient: + pymongo.MongoClient = MockMongoClient + if pymongo.MongoReplicaSetClient is not MockMongoClient: + pymongo.MongoClient = MockMongoClient + + +class MyTransformer(mongo.BaseTransform): + """Added here just to check manipulator logic is used correctly.""" + + def transform_incoming(self, son, collection): + return super(MyTransformer, self).transform_incoming(son, collection) + + def transform_outgoing(self, son, collection): + return super(MyTransformer, self).transform_outgoing(son, collection) + + +class MongoCache(tests.BaseTestCase): + def setUp(self): + super(MongoCache, self).setUp() + global COLLECTIONS + COLLECTIONS = {} + mongo.MongoApi._DB = {} + mongo.MongoApi._MONGO_COLLS = {} + pymongo_override() + # using typical configuration + self.arguments = { + 'db_hosts': 'localhost:27017', + 'db_name': 'ks_cache', + 'cache_collection': 'cache', + 'username': 'test_user', + 'password': 'test_password' + } + + def test_missing_db_hosts(self): + self.arguments.pop('db_hosts') + region = dp_region.make_region() + self.assertRaises(exception.ValidationError, region.configure, + 'keystone.cache.mongo', + arguments=self.arguments) + + def test_missing_db_name(self): + self.arguments.pop('db_name') + region = dp_region.make_region() + self.assertRaises(exception.ValidationError, region.configure, + 'keystone.cache.mongo', + arguments=self.arguments) + + def test_missing_cache_collection_name(self): + self.arguments.pop('cache_collection') + region = dp_region.make_region() + self.assertRaises(exception.ValidationError, region.configure, + 'keystone.cache.mongo', + arguments=self.arguments) + + def test_incorrect_write_concern(self): + self.arguments['w'] = 'one value' + region = dp_region.make_region() + self.assertRaises(exception.ValidationError, region.configure, + 'keystone.cache.mongo', + arguments=self.arguments) + + def test_correct_write_concern(self): + self.arguments['w'] = 1 + region = dp_region.make_region().configure('keystone.cache.mongo', + arguments=self.arguments) + + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue10") + # There is no proxy so can access MongoCacheBackend directly + self.assertEqual(1, region.backend.api.w) + + def test_incorrect_read_preference(self): + self.arguments['read_preference'] = 'inValidValue' + region = dp_region.make_region().configure('keystone.cache.mongo', + arguments=self.arguments) + # As per delayed loading of pymongo, read_preference value should + # still be string and NOT enum + self.assertEqual('inValidValue', region.backend.api.read_preference) + + random_key = uuid.uuid4().hex + self.assertRaises(ValueError, region.set, + random_key, "dummyValue10") + + def test_correct_read_preference(self): + self.arguments['read_preference'] = 'secondaryPreferred' + region = dp_region.make_region().configure('keystone.cache.mongo', + arguments=self.arguments) + # As per delayed loading of pymongo, read_preference value should + # still be string and NOT enum + self.assertEqual('secondaryPreferred', + region.backend.api.read_preference) + + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue10") + + # Now as pymongo is loaded so expected read_preference value is enum. + # There is no proxy so can access MongoCacheBackend directly + self.assertEqual(3, region.backend.api.read_preference) + + def test_missing_replica_set_name(self): + self.arguments['use_replica'] = True + region = dp_region.make_region() + self.assertRaises(exception.ValidationError, region.configure, + 'keystone.cache.mongo', + arguments=self.arguments) + + def test_provided_replica_set_name(self): + self.arguments['use_replica'] = True + self.arguments['replicaset_name'] = 'my_replica' + dp_region.make_region().configure('keystone.cache.mongo', + arguments=self.arguments) + self.assertTrue(True) # reached here means no initialization error + + def test_incorrect_mongo_ttl_seconds(self): + self.arguments['mongo_ttl_seconds'] = 'sixty' + region = dp_region.make_region() + self.assertRaises(exception.ValidationError, region.configure, + 'keystone.cache.mongo', + arguments=self.arguments) + + def test_cache_configuration_values_assertion(self): + self.arguments['use_replica'] = True + self.arguments['replicaset_name'] = 'my_replica' + self.arguments['mongo_ttl_seconds'] = 60 + self.arguments['ssl'] = False + region = dp_region.make_region().configure('keystone.cache.mongo', + arguments=self.arguments) + # There is no proxy so can access MongoCacheBackend directly + self.assertEqual('localhost:27017', region.backend.api.hosts) + self.assertEqual('ks_cache', region.backend.api.db_name) + self.assertEqual('cache', region.backend.api.cache_collection) + self.assertEqual('test_user', region.backend.api.username) + self.assertEqual('test_password', region.backend.api.password) + self.assertEqual(True, region.backend.api.use_replica) + self.assertEqual('my_replica', region.backend.api.replicaset_name) + self.assertEqual(False, region.backend.api.conn_kwargs['ssl']) + self.assertEqual(60, region.backend.api.ttl_seconds) + + def test_multiple_region_cache_configuration(self): + arguments1 = copy.copy(self.arguments) + arguments1['cache_collection'] = 'cache_region1' + + region1 = dp_region.make_region().configure('keystone.cache.mongo', + arguments=arguments1) + # There is no proxy so can access MongoCacheBackend directly + self.assertEqual('localhost:27017', region1.backend.api.hosts) + self.assertEqual('ks_cache', region1.backend.api.db_name) + self.assertEqual('cache_region1', region1.backend.api.cache_collection) + self.assertEqual('test_user', region1.backend.api.username) + self.assertEqual('test_password', region1.backend.api.password) + # Should be None because of delayed initialization + self.assertIsNone(region1.backend.api._data_manipulator) + + random_key1 = uuid.uuid4().hex + region1.set(random_key1, "dummyValue10") + self.assertEqual("dummyValue10", region1.get(random_key1)) + # Now should have initialized + self.assertIsInstance(region1.backend.api._data_manipulator, + mongo.BaseTransform) + + class_name = '%s.%s' % (MyTransformer.__module__, "MyTransformer") + + arguments2 = copy.copy(self.arguments) + arguments2['cache_collection'] = 'cache_region2' + arguments2['son_manipulator'] = class_name + + region2 = dp_region.make_region().configure('keystone.cache.mongo', + arguments=arguments2) + # There is no proxy so can access MongoCacheBackend directly + self.assertEqual('localhost:27017', region2.backend.api.hosts) + self.assertEqual('ks_cache', region2.backend.api.db_name) + self.assertEqual('cache_region2', region2.backend.api.cache_collection) + + # Should be None because of delayed initialization + self.assertIsNone(region2.backend.api._data_manipulator) + + random_key = uuid.uuid4().hex + region2.set(random_key, "dummyValue20") + self.assertEqual("dummyValue20", region2.get(random_key)) + # Now should have initialized + self.assertIsInstance(region2.backend.api._data_manipulator, + MyTransformer) + + region1.set(random_key1, "dummyValue22") + self.assertEqual("dummyValue22", region1.get(random_key1)) + + def test_typical_configuration(self): + + dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + self.assertTrue(True) # reached here means no initialization error + + def test_backend_get_missing_data(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + + random_key = uuid.uuid4().hex + # should return NO_VALUE as key does not exist in cache + self.assertEqual(api.NO_VALUE, region.get(random_key)) + + def test_backend_set_data(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", region.get(random_key)) + + def test_backend_set_data_with_string_as_valid_ttl(self): + + self.arguments['mongo_ttl_seconds'] = '3600' + region = dp_region.make_region().configure('keystone.cache.mongo', + arguments=self.arguments) + self.assertEqual(3600, region.backend.api.ttl_seconds) + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", region.get(random_key)) + + def test_backend_set_data_with_int_as_valid_ttl(self): + + self.arguments['mongo_ttl_seconds'] = 1800 + region = dp_region.make_region().configure('keystone.cache.mongo', + arguments=self.arguments) + self.assertEqual(1800, region.backend.api.ttl_seconds) + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", region.get(random_key)) + + def test_backend_set_none_as_data(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + + random_key = uuid.uuid4().hex + region.set(random_key, None) + self.assertIsNone(region.get(random_key)) + + def test_backend_set_blank_as_data(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + + random_key = uuid.uuid4().hex + region.set(random_key, "") + self.assertEqual("", region.get(random_key)) + + def test_backend_set_same_key_multiple_times(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", region.get(random_key)) + + dict_value = {'key1': 'value1'} + region.set(random_key, dict_value) + self.assertEqual(dict_value, region.get(random_key)) + + region.set(random_key, "dummyValue2") + self.assertEqual("dummyValue2", region.get(random_key)) + + def test_backend_multi_set_data(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + random_key = uuid.uuid4().hex + random_key1 = uuid.uuid4().hex + random_key2 = uuid.uuid4().hex + random_key3 = uuid.uuid4().hex + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(api.NO_VALUE, region.get(random_key)) + self.assertFalse(region.get(random_key)) + self.assertEqual("dummyValue1", region.get(random_key1)) + self.assertEqual("dummyValue2", region.get(random_key2)) + self.assertEqual("dummyValue3", region.get(random_key3)) + + def test_backend_multi_get_data(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + random_key = uuid.uuid4().hex + random_key1 = uuid.uuid4().hex + random_key2 = uuid.uuid4().hex + random_key3 = uuid.uuid4().hex + mapping = {random_key1: 'dummyValue1', + random_key2: '', + random_key3: 'dummyValue3'} + region.set_multi(mapping) + + keys = [random_key, random_key1, random_key2, random_key3] + results = region.get_multi(keys) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(api.NO_VALUE, results[0]) + self.assertEqual("dummyValue1", results[1]) + self.assertEqual("", results[2]) + self.assertEqual("dummyValue3", results[3]) + + def test_backend_multi_set_should_update_existing(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + random_key = uuid.uuid4().hex + random_key1 = uuid.uuid4().hex + random_key2 = uuid.uuid4().hex + random_key3 = uuid.uuid4().hex + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(api.NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue1", region.get(random_key1)) + self.assertEqual("dummyValue2", region.get(random_key2)) + self.assertEqual("dummyValue3", region.get(random_key3)) + + mapping = {random_key1: 'dummyValue4', + random_key2: 'dummyValue5'} + region.set_multi(mapping) + self.assertEqual(api.NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue4", region.get(random_key1)) + self.assertEqual("dummyValue5", region.get(random_key2)) + self.assertEqual("dummyValue3", region.get(random_key3)) + + def test_backend_multi_set_get_with_blanks_none(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + random_key = uuid.uuid4().hex + random_key1 = uuid.uuid4().hex + random_key2 = uuid.uuid4().hex + random_key3 = uuid.uuid4().hex + random_key4 = uuid.uuid4().hex + mapping = {random_key1: 'dummyValue1', + random_key2: None, + random_key3: '', + random_key4: 'dummyValue4'} + region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(api.NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue1", region.get(random_key1)) + self.assertIsNone(region.get(random_key2)) + self.assertEqual("", region.get(random_key3)) + self.assertEqual("dummyValue4", region.get(random_key4)) + + keys = [random_key, random_key1, random_key2, random_key3, random_key4] + results = region.get_multi(keys) + + # should return NO_VALUE as key does not exist in cache + self.assertEqual(api.NO_VALUE, results[0]) + self.assertEqual("dummyValue1", results[1]) + self.assertIsNone(results[2]) + self.assertEqual("", results[3]) + self.assertEqual("dummyValue4", results[4]) + + mapping = {random_key1: 'dummyValue5', + random_key2: 'dummyValue6'} + region.set_multi(mapping) + self.assertEqual(api.NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue5", region.get(random_key1)) + self.assertEqual("dummyValue6", region.get(random_key2)) + self.assertEqual("", region.get(random_key3)) + + def test_backend_delete_data(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", region.get(random_key)) + + region.delete(random_key) + # should return NO_VALUE as key no longer exists in cache + self.assertEqual(api.NO_VALUE, region.get(random_key)) + + def test_backend_multi_delete_data(self): + + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + random_key = uuid.uuid4().hex + random_key1 = uuid.uuid4().hex + random_key2 = uuid.uuid4().hex + random_key3 = uuid.uuid4().hex + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(api.NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue1", region.get(random_key1)) + self.assertEqual("dummyValue2", region.get(random_key2)) + self.assertEqual("dummyValue3", region.get(random_key3)) + self.assertEqual(api.NO_VALUE, region.get("InvalidKey")) + + keys = mapping.keys() + + region.delete_multi(keys) + + self.assertEqual(api.NO_VALUE, region.get("InvalidKey")) + # should return NO_VALUE as keys no longer exist in cache + self.assertEqual(api.NO_VALUE, region.get(random_key1)) + self.assertEqual(api.NO_VALUE, region.get(random_key2)) + self.assertEqual(api.NO_VALUE, region.get(random_key3)) + + def test_additional_crud_method_arguments_support(self): + """Additional arguments should works across find/insert/update.""" + + self.arguments['wtimeout'] = 30000 + self.arguments['j'] = True + self.arguments['continue_on_error'] = True + self.arguments['secondary_acceptable_latency_ms'] = 60 + region = dp_region.make_region().configure( + 'keystone.cache.mongo', + arguments=self.arguments + ) + + # There is no proxy so can access MongoCacheBackend directly + api_methargs = region.backend.api.meth_kwargs + self.assertEqual(30000, api_methargs['wtimeout']) + self.assertEqual(True, api_methargs['j']) + self.assertEqual(True, api_methargs['continue_on_error']) + self.assertEqual(60, api_methargs['secondary_acceptable_latency_ms']) + + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue1") + self.assertEqual("dummyValue1", region.get(random_key)) + + region.set(random_key, "dummyValue2") + self.assertEqual("dummyValue2", region.get(random_key)) + + random_key = uuid.uuid4().hex + region.set(random_key, "dummyValue3") + self.assertEqual("dummyValue3", region.get(random_key)) |