diff options
Diffstat (limited to 'old/python_moonutilities/python_moonutilities/cache.py')
| -rw-r--r-- | old/python_moonutilities/python_moonutilities/cache.py | 703 |
1 files changed, 703 insertions, 0 deletions
diff --git a/old/python_moonutilities/python_moonutilities/cache.py b/old/python_moonutilities/python_moonutilities/cache.py new file mode 100644 index 00000000..49a3ef5b --- /dev/null +++ b/old/python_moonutilities/python_moonutilities/cache.py @@ -0,0 +1,703 @@ +import logging +import time +import python_moonutilities.request_wrapper as requests +from uuid import uuid4 +from python_moonutilities import configuration, exceptions + +logger = logging.getLogger("moon.utilities.cache") + + +class Cache(object): + # TODO (asteroide): set cache integer in CONF file + ''' + [NOTE] Propose to define the following variables inside the init method + as defining them out side the init, will be treated as private static variables + and keep tracks with any changes done anywhere + for more info : you can check https://docs.python.org/3/tutorial/classes.html#class-and-instance-variables + ''' + __UPDATE_INTERVAL = 10 + + __CONTAINERS = {} + __CONTAINERS_UPDATE = 0 + + __CONTAINER_CHAINING_UPDATE = 0 + __CONTAINER_CHAINING = {} + + __PDP = {} + __PDP_UPDATE = 0 + + __POLICIES = {} + __POLICIES_UPDATE = 0 + + __MODELS = {} + __MODELS_UPDATE = 0 + + __SUBJECTS = {} + __OBJECTS = {} + __ACTIONS = {} + + __SUBJECT_ASSIGNMENTS = {} + __OBJECT_ASSIGNMENTS = {} + __ACTION_ASSIGNMENTS = {} + + __SUBJECT_CATEGORIES = {} + __SUBJECT_CATEGORIES_UPDATE = 0 + __OBJECT_CATEGORIES = {} + __OBJECT_CATEGORIES_UPDATE = 0 + __ACTION_CATEGORIES = {} + __ACTION_CATEGORIES_UPDATE = 0 + + __META_RULES = {} + __META_RULES_UPDATE = 0 + + __RULES = {} + __RULES_UPDATE = 0 + + __AUTHZ_REQUESTS = {} + + def __init__(self): + self.manager_url = "{}://{}:{}".format( + configuration.get_components()['manager'].get('protocol', 'http'), + configuration.get_components()['manager']['hostname'], + configuration.get_components()['manager']['port'] + ) + self.orchestrator_url = "{}://{}:{}".format( + configuration.get_components()['orchestrator'].get('protocol', 'http'), + configuration.get_components()['orchestrator']['hostname'], + configuration.get_components()['orchestrator']['port'] + ) + + def update(self): + self.__update_container() + self.__update_pdp() + self.__update_policies() + self.__update_models() + for key, value in self.__PDP.items(): + # LOG.info("Updating container_chaining with {}".format(value["keystone_project_id"])) + if "keystone_project_id" in value: + self.__update_container_chaining(value["keystone_project_id"]) + else: + logger.warning("no 'keystone_project_id' found while Updating container_chaining") + + @property + def authz_requests(self): + return self.__AUTHZ_REQUESTS + + # perimeter functions + + @property + def subjects(self): + return self.__SUBJECTS + + def __update_subjects(self, policy_id): + response = requests.get("{}/policies/{}/subjects".format(self.manager_url, policy_id)) + if 'subjects' in response.json(): + self.__SUBJECTS[policy_id] = response.json()['subjects'] + else: + raise exceptions.SubjectUnknown("Cannot find subject within policy_id {}".format(policy_id)) + + def get_subject(self, policy_id, name): + if not policy_id: + raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) + + if policy_id in self.subjects: + for _subject_id, _subject_dict in self.subjects[policy_id].items(): + if _subject_id == name or _subject_dict.get("name") == name: + return _subject_id + + self.__update_subjects(policy_id) + + if policy_id in self.subjects: + for _subject_id, _subject_dict in self.subjects[policy_id].items(): + if _subject_id == name or _subject_dict.get("name") == name: + return _subject_id + + raise exceptions.SubjectUnknown("Cannot find subject {}".format(name)) + + @property + def objects(self): + return self.__OBJECTS + + def __update_objects(self, policy_id): + response = requests.get("{}/policies/{}/objects".format(self.manager_url, policy_id)) + if 'objects' in response.json(): + self.__OBJECTS[policy_id] = response.json()['objects'] + else: + raise exceptions.ObjectUnknown("Cannot find object within policy_id {}".format(policy_id)) + + def get_object(self, policy_id, name): + if not policy_id: + raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) + + if policy_id in self.objects: + for _object_id, _object_dict in self.__OBJECTS[policy_id].items(): + if _object_id == name or _object_dict.get("name") == name: + return _object_id + + self.__update_objects(policy_id) + + if policy_id in self.objects: + for _object_id, _object_dict in self.__OBJECTS[policy_id].items(): + if _object_id == name or _object_dict.get("name") == name: + return _object_id + + raise exceptions.ObjectUnknown("Cannot find object {}".format(name)) + + @property + def actions(self): + return self.__ACTIONS + + def __update_actions(self, policy_id): + response = requests.get("{}/policies/{}/actions".format(self.manager_url, policy_id)) + + if 'actions' in response.json(): + self.__ACTIONS[policy_id] = response.json()['actions'] + else: + raise exceptions.ActionUnknown("Cannot find action within policy_id {}".format(policy_id)) + + def get_action(self, policy_id, name): + if not policy_id: + raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) + + if policy_id in self.actions: + for _action_id, _action_dict in self.__ACTIONS[policy_id].items(): + if _action_id == name or _action_dict.get("name") == name: + return _action_id + + self.__update_actions(policy_id) + + for _action_id, _action_dict in self.__ACTIONS[policy_id].items(): + if _action_id == name or _action_dict.get("name") == name: + return _action_id + + raise exceptions.ActionUnknown("Cannot find action {}".format(name)) + + # meta_rule functions + + @property + def meta_rules(self): + current_time = time.time() + if self.__META_RULES_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__META_RULES_UPDATE = current_time + self.__update_meta_rules() + self.__META_RULES_UPDATE = current_time + return self.__META_RULES + + def __update_meta_rules(self): + response = requests.get("{}/meta_rules".format(self.manager_url)) + + if 'meta_rules' in response.json(): + self.__META_RULES = response.json()['meta_rules'] + else: + raise exceptions.MetaRuleUnknown("Cannot find meta rules") + + # rule functions + + @property + def rules(self): + current_time = time.time() + if self.__RULES_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__RULES_UPDATE = current_time + self.__update_rules() + self.__RULES_UPDATE = current_time + return self.__RULES + + def __update_rules(self): + for policy_id in self.policies: + logger.debug("Get {}".format("{}/policies/{}/rules".format( + self.manager_url, policy_id))) + + response = requests.get("{}/policies/{}/rules".format( + self.manager_url, policy_id)) + if 'rules' in response.json(): + self.__RULES[policy_id] = response.json()['rules'] + else: + logger.warning(" no 'rules' found within policy_id: {}".format(policy_id)) + + logger.debug("UPDATE RULES {}".format(self.__RULES)) + + # assignment functions + + def update_assignments(self, policy_id=None, perimeter_id=None): + if policy_id: + self.__update_subject_assignments(policy_id=policy_id, perimeter_id=perimeter_id) + self.__update_object_assignments(policy_id=policy_id, perimeter_id=perimeter_id) + self.__update_action_assignments(policy_id=policy_id, perimeter_id=perimeter_id) + else: + for policy_id in self.__POLICIES: + self.__update_subject_assignments(policy_id=policy_id, perimeter_id=perimeter_id) + self.__update_object_assignments(policy_id=policy_id, perimeter_id=perimeter_id) + self.__update_action_assignments(policy_id=policy_id, perimeter_id=perimeter_id) + + @property + def subject_assignments(self): + return self.__SUBJECT_ASSIGNMENTS + + def __update_subject_assignments(self, policy_id, perimeter_id=None): + if perimeter_id: + response = requests.get("{}/policies/{}/subject_assignments/{}".format( + self.manager_url, policy_id, perimeter_id)) + else: + response = requests.get("{}/policies/{}/subject_assignments".format( + self.manager_url, policy_id)) + + if 'subject_assignments' in response.json(): + if policy_id not in self.subject_assignments: + self.__SUBJECT_ASSIGNMENTS[policy_id] = {} + self.__SUBJECT_ASSIGNMENTS[policy_id] = response.json()['subject_assignments'] + else: + raise exceptions.SubjectAssignmentUnknown( + "Cannot find subject assignment within policy_id {}".format(policy_id)) + + def get_subject_assignments(self, policy_id, perimeter_id, category_id): + if not policy_id: + raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) + + if policy_id not in self.subject_assignments: + self.__update_subject_assignments(policy_id, perimeter_id) + + for key, value in self.subject_assignments[policy_id].items(): + if all(k in value for k in ("subject_id", "category_id", "assignments")): + if perimeter_id == value['subject_id'] and category_id == value['category_id']: + return value['assignments'] + else: + logger.warning("'subject_id' or 'category_id' or 'assignments'" + " keys are not found in subject_assignments") + return [] + + @property + def object_assignments(self): + return self.__OBJECT_ASSIGNMENTS + + def __update_object_assignments(self, policy_id, perimeter_id=None): + if perimeter_id: + response = requests.get("{}/policies/{}/object_assignments/{}".format( + self.manager_url, policy_id, perimeter_id)) + else: + response = requests.get("{}/policies/{}/object_assignments".format( + self.manager_url, policy_id)) + + if 'object_assignments' in response.json(): + if policy_id not in self.object_assignments: + self.__OBJECT_ASSIGNMENTS[policy_id] = {} + + self.__OBJECT_ASSIGNMENTS[policy_id] = response.json()['object_assignments'] + else: + raise exceptions.ObjectAssignmentUnknown( + "Cannot find object assignment within policy_id {}".format(policy_id)) + + def get_object_assignments(self, policy_id, perimeter_id, category_id): + if not policy_id: + raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) + + if policy_id not in self.object_assignments: + self.__update_object_assignments(policy_id, perimeter_id) + + for key, value in self.object_assignments[policy_id].items(): + if all(k in value for k in ("object_id", "category_id", "assignments")): + if perimeter_id == value['object_id'] and category_id == value['category_id']: + return value['assignments'] + else: + logger.warning("'object_id' or 'category_id' or'assignments'" + " keys are not found in object_assignments") + return [] + + @property + def action_assignments(self): + return self.__ACTION_ASSIGNMENTS + + def __update_action_assignments(self, policy_id, perimeter_id=None): + if perimeter_id: + response = requests.get("{}/policies/{}/action_assignments/{}".format( + self.manager_url, policy_id, perimeter_id)) + else: + response = requests.get("{}/policies/{}/action_assignments".format( + self.manager_url, policy_id)) + + if 'action_assignments' in response.json(): + if policy_id not in self.__ACTION_ASSIGNMENTS: + self.__ACTION_ASSIGNMENTS[policy_id] = {} + + self.__ACTION_ASSIGNMENTS[policy_id] = response.json()['action_assignments'] + else: + raise exceptions.ActionAssignmentUnknown( + "Cannot find action assignment within policy_id {}".format(policy_id)) + + def get_action_assignments(self, policy_id, perimeter_id, category_id): + if not policy_id: + raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) + + if policy_id not in self.action_assignments: + self.__update_action_assignments(policy_id, perimeter_id) + + for key, value in self.action_assignments[policy_id].items(): + if all(k in value for k in ("action_id", "category_id", "assignments")): + if perimeter_id == value['action_id'] and category_id == value['category_id']: + return value['assignments'] + else: + logger.warning("'action_id' or 'category_id' or'assignments'" + " keys are not found in action_assignments") + return [] + + # category functions + + @property + def subject_categories(self): + current_time = time.time() + if self.__SUBJECT_CATEGORIES_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__SUBJECT_CATEGORIES_UPDATE = current_time + self.__update_subject_categories() + self.__SUBJECT_CATEGORIES_UPDATE = current_time + return self.__SUBJECT_CATEGORIES + + def __update_subject_categories(self): + response = requests.get("{}/policies/subject_categories".format( + self.manager_url)) + + if 'subject_categories' in response.json(): + self.__SUBJECT_CATEGORIES.update(response.json()['subject_categories']) + else: + raise exceptions.SubjectCategoryUnknown("Cannot find subject category") + + @property + def object_categories(self): + current_time = time.time() + if self.__OBJECT_CATEGORIES_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__OBJECT_CATEGORIES_UPDATE = current_time + self.__update_object_categories() + self.__OBJECT_CATEGORIES_UPDATE = current_time + return self.__OBJECT_CATEGORIES + + def __update_object_categories(self): + response = requests.get("{}/policies/object_categories".format(self.manager_url)) + + if 'object_categories' in response.json(): + self.__OBJECT_CATEGORIES.update(response.json()['object_categories']) + else: + raise exceptions.ObjectCategoryUnknown("Cannot find object category") + + @property + def action_categories(self): + current_time = time.time() + if self.__ACTION_CATEGORIES_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__ACTION_CATEGORIES_UPDATE = current_time + self.__update_action_categories() + self.__ACTION_CATEGORIES_UPDATE = current_time + return self.__ACTION_CATEGORIES + + def __update_action_categories(self): + response = requests.get("{}/policies/action_categories".format(self.manager_url)) + + if 'action_categories' in response.json(): + self.__ACTION_CATEGORIES.update(response.json()['action_categories']) + else: + raise exceptions.ActionCategoryUnknown("Cannot find action category") + + # PDP functions + + def __update_pdp(self): + response = requests.get("{}/pdp".format(self.manager_url)) + pdp = response.json() + if 'pdps' in pdp: + for _pdp in pdp["pdps"].values(): + if "keystone_project_id" in _pdp and _pdp['keystone_project_id'] not in self.container_chaining: + self.__CONTAINER_CHAINING[_pdp['keystone_project_id']] = {} + # Note (asteroide): force update of chaining + self.__update_container_chaining(_pdp['keystone_project_id']) + for key, value in pdp["pdps"].items(): + self.__PDP[key] = value + + else: + raise exceptions.PdpError("Cannot find 'pdps' key") + + @property + def pdp(self): + """Policy Decision Point + Example of content: + { + "pdp_id": { + "keystone_project_id": "keystone_project_id", + "name": "pdp1", + "description": "test", + "security_pipeline": [ + "policy_id" + ] + } + } + + :return: + """ + current_time = time.time() + if self.__PDP_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__PDP_UPDATE = current_time + self.__update_pdp() + self.__PDP_UPDATE = current_time + return self.__PDP + + # policy functions + def __update_policies(self): + response = requests.get("{}/policies".format(self.manager_url)) + policies = response.json() + + if 'policies' in policies: + for key, value in policies["policies"].items(): + self.__POLICIES[key] = value + else: + raise exceptions.PolicytNotFound("Cannot find 'policies' key") + + @property + def policies(self): + current_time = time.time() + if self.__POLICIES_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__POLICIES_UPDATE = current_time + self.__update_policies() + self.__POLICIES_UPDATE = current_time + return self.__POLICIES + + # model functions + + def __update_models(self): + response = requests.get("{}/models".format(self.manager_url)) + models = response.json() + if 'models' in models: + for key, value in models["models"].items(): + self.__MODELS[key] = value + else: + raise exceptions.ModelNotFound("Cannot find 'models' key") + + @property + def models(self): + current_time = time.time() + if self.__MODELS_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__MODELS_UPDATE = current_time + self.__update_models() + self.__MODELS_UPDATE = current_time + return self.__MODELS + + # helper functions + + def get_policy_from_meta_rules(self, meta_rule_id): + for pdp_key, pdp_value in self.pdp.items(): + if "security_pipeline" in pdp_value: + for policy_id in pdp_value["security_pipeline"]: + if policy_id in self.policies and "model_id" in self.policies[policy_id]: + model_id = self.policies[policy_id]["model_id"] + if model_id in self.models and "meta_rules" in self.models[model_id]: + if meta_rule_id in self.models[model_id]["meta_rules"]: + return policy_id + else: + logger.warning( + "Cannot find model_id: {} within " + "models and 'meta_rules' key".format(model_id)) + else: + logger.warning( + "Cannot find policy_id: {} " + "within policies and 'model_id' key".format( + policy_id)) + else: + logger.warning("Cannot find 'security_pipeline' " + "key within pdp ") + + def get_meta_rule_ids_from_pdp_value(self, pdp_value): + meta_rules = [] + if "security_pipeline" in pdp_value: + for policy_id in pdp_value["security_pipeline"]: + if policy_id not in self.policies or "model_id" not in self.policies[policy_id]: + raise exceptions.PolicyUnknown("Cannot find 'models' key") + model_id = self.policies[policy_id]["model_id"] + if model_id not in self.models or 'meta_rules' not in self.models[model_id]: + raise exceptions.ModelNotFound("Cannot find 'models' key") + for meta_rule in self.models[model_id]["meta_rules"]: + meta_rules.append(meta_rule) + return meta_rules + raise exceptions.PdpContentError + + def get_pdp_from_keystone_project(self, keystone_project_id): + for pdp_key, pdp_value in self.pdp.items(): + if "keystone_project_id" in pdp_value and \ + keystone_project_id == pdp_value["keystone_project_id"]: + return pdp_key + + def get_keystone_project_id_from_policy_id(self, policy_id): + for pdp_key, pdp_value in self.pdp.items(): + if "security_pipeline" in pdp_value and \ + "keystone_project_id" in pdp_value: + if policy_id in pdp_value["security_pipeline"]: + return pdp_value["keystone_project_id"] + else: + logger.warning(" 'security_pipeline','keystone_project_id' " + "key not in pdp {}".format(pdp_value)) + + def get_keystone_project_id_from_pdp_id(self, pdp_id): + if pdp_id in self.pdp: + pdp_value = self.pdp.get(pdp_id) + if "security_pipeline" in pdp_value and \ + "keystone_project_id" in pdp_value: + return pdp_value["keystone_project_id"] + logger.warning("Unknown PDP ID".format(pdp_id)) + + def get_containers_from_keystone_project_id(self, keystone_project_id, + meta_rule_id=None): + for container_id, container_values in self.containers.items(): + for container_value in container_values: + if 'keystone_project_id' not in container_value: + continue + if container_value['keystone_project_id'] == keystone_project_id: + if not meta_rule_id: + yield container_id, container_value + elif "meta_rule_id" in container_value and \ + container_value.get('meta_rule_id') == meta_rule_id: + yield container_id, container_value + break + + # containers functions + + def __update_container(self): + response = requests.get("{}/pods".format(self.orchestrator_url)) + pods = response.json() + if "pods" in pods: + for key, value in pods["pods"].items(): + # if key not in self.__CONTAINERS: + self.__CONTAINERS[key] = value + # else: + # for container in value: + # self.__CONTAINERS[key].update(value) + else: + raise exceptions.PodError("Cannot find 'pods' key") + + def add_container(self, container_data): + """Add a new container in the cache + + :param container_data: dictionary with information for the container + Example: + { + "name": name, + "hostname": name, + "port": { + "PrivatePort": tcp_port, + "Type": "tcp", + "IP": "0.0.0.0", + "PublicPort": tcp_port + }, + "keystone_project_id": uuid, + "pdp_id": self.CACHE.get_pdp_from_keystone_project(uuid), + "meta_rule_id": meta_rule_id, + "container_name": container_name, + "plugin_name": plugin_name + "container_id": "container_id" + } + + :return: + """ + if all(k in container_data for k in ("keystone_project_id", "name", "container_id", "policy_id", + "meta_rule_id", "port")) \ + and all(k in container_data['port'] for k in ("PublicPort", "Type", "IP", "PrivatePort")): + + self.__CONTAINERS[uuid4().hex] = { + "keystone_project_id": container_data['keystone_project_id'], + "name": container_data['name'], + "container_id": container_data['container_id'], + "hostname": container_data['name'], + "policy_id": container_data['policy_id'], + "meta_rule_id": container_data['meta_rule_id'], + "port": [ + { + "PublicPort": container_data['port']["PublicPort"], + "Type": container_data['port']["Type"], + "IP": container_data['port']["IP"], + "PrivatePort": container_data['port']["PrivatePort"] + } + ], + "genre": container_data['plugin_name'] + } + self.__update_container_chaining(self.get_keystone_project_id_from_policy_id(container_data['policy_id'])) + else: + raise exceptions.ContainerError("Cannot find 'container' parameters key") + + @property + def containers(self): + """Containers cache + example of content : + { + "policy_uuid1": "component_hostname1", + "policy_uuid2": "component_hostname2", + } + :return: + """ + current_time = time.time() + if self.__CONTAINERS_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__CONTAINERS_UPDATE = current_time + self.__update_container() + self.__CONTAINERS_UPDATE = current_time + return self.__CONTAINERS + + @property + def container_chaining(self): + """Cache for mapping Keystone Project ID with meta_rule ID and container ID + Example of content: + { + "keystone_project_id": [ + { + "container_id": "container_id", + "genre": "genre", + "policy_id": "policy_id", + "meta_rule_id": "meta_rule_id", + } + ] + } + + :return: + """ + current_time = time.time() + if self.__CONTAINER_CHAINING_UPDATE + self.__UPDATE_INTERVAL < current_time: + self.__CONTAINER_CHAINING_UPDATE = current_time + for key, value in self.pdp.items(): + if "keystone_project_id" in value: + if not value["keystone_project_id"]: + continue + self.__update_container_chaining(value["keystone_project_id"]) + else: + logger.warning("no 'keystone_project_id' found") + self.__CONTAINER_CHAINING_UPDATE = current_time + return self.__CONTAINER_CHAINING + + def __update_container_chaining(self, keystone_project_id): + container_ids = [] + for pdp_id, pdp_value, in self.__PDP.items(): + if pdp_value: + if all(k in pdp_value for k in ("keystone_project_id", "security_pipeline")) \ + and pdp_value["keystone_project_id"] == keystone_project_id: + for policy_id in pdp_value["security_pipeline"]: + if policy_id in self.policies and "model_id" in self.policies[policy_id]: + model_id = self.policies[policy_id]['model_id'] + if model_id in self.models and "meta_rules" in self.models[model_id]: + for meta_rule_id in self.models[model_id]["meta_rules"]: + for container_id, container_value in self.get_containers_from_keystone_project_id( + keystone_project_id, + meta_rule_id + ): + if "name" in container_value: + if all(k in container_value for k in ("genre", "port")): + container_ids.append( + { + "container_id": container_value["name"], + "genre": container_value["genre"], + "policy_id": policy_id, + "meta_rule_id": meta_rule_id, + "hostname": container_value["name"], + "hostip": "127.0.0.1", + "port": container_value["port"], + } + ) + else: + logger.warning("Container content keys not found {}", container_value) + else: + logger.warning("Container content keys not found {}", container_value) + else: + raise exceptions.ModelUnknown("Cannot find model_id: {} in models and " + "may not contains 'meta_rules' key".format(model_id)) + else: + raise exceptions.PolicyUnknown("Cannot find policy within policy_id: {}, " + "and may not contains 'model_id' key".format(policy_id)) + + self.__CONTAINER_CHAINING[keystone_project_id] = container_ids |