diff options
Diffstat (limited to 'python_moonutilities/python_moonutilities/cache.py')
| -rw-r--r-- | python_moonutilities/python_moonutilities/cache.py | 703 |
1 files changed, 0 insertions, 703 deletions
diff --git a/python_moonutilities/python_moonutilities/cache.py b/python_moonutilities/python_moonutilities/cache.py deleted file mode 100644 index 49a3ef5b..00000000 --- a/python_moonutilities/python_moonutilities/cache.py +++ /dev/null @@ -1,703 +0,0 @@ -import logging -import time -import python_moonutilities.request_wrapper as requests -from uuid import uuid4 -from python_moonutilities import configuration, exceptions - -logger = logging.getLogger("moon.utilities.cache") - - -class Cache(object): - # TODO (asteroide): set cache integer in CONF file - ''' - [NOTE] Propose to define the following variables inside the init method - as defining them out side the init, will be treated as private static variables - and keep tracks with any changes done anywhere - for more info : you can check https://docs.python.org/3/tutorial/classes.html#class-and-instance-variables - ''' - __UPDATE_INTERVAL = 10 - - __CONTAINERS = {} - __CONTAINERS_UPDATE = 0 - - __CONTAINER_CHAINING_UPDATE = 0 - __CONTAINER_CHAINING = {} - - __PDP = {} - __PDP_UPDATE = 0 - - __POLICIES = {} - __POLICIES_UPDATE = 0 - - __MODELS = {} - __MODELS_UPDATE = 0 - - __SUBJECTS = {} - __OBJECTS = {} - __ACTIONS = {} - - __SUBJECT_ASSIGNMENTS = {} - __OBJECT_ASSIGNMENTS = {} - __ACTION_ASSIGNMENTS = {} - - __SUBJECT_CATEGORIES = {} - __SUBJECT_CATEGORIES_UPDATE = 0 - __OBJECT_CATEGORIES = {} - __OBJECT_CATEGORIES_UPDATE = 0 - __ACTION_CATEGORIES = {} - __ACTION_CATEGORIES_UPDATE = 0 - - __META_RULES = {} - __META_RULES_UPDATE = 0 - - __RULES = {} - __RULES_UPDATE = 0 - - __AUTHZ_REQUESTS = {} - - def __init__(self): - self.manager_url = "{}://{}:{}".format( - configuration.get_components()['manager'].get('protocol', 'http'), - configuration.get_components()['manager']['hostname'], - configuration.get_components()['manager']['port'] - ) - self.orchestrator_url = "{}://{}:{}".format( - configuration.get_components()['orchestrator'].get('protocol', 'http'), - configuration.get_components()['orchestrator']['hostname'], - configuration.get_components()['orchestrator']['port'] - ) - - def update(self): - self.__update_container() - self.__update_pdp() - self.__update_policies() - self.__update_models() - for key, value in self.__PDP.items(): - # LOG.info("Updating container_chaining with {}".format(value["keystone_project_id"])) - if "keystone_project_id" in value: - self.__update_container_chaining(value["keystone_project_id"]) - else: - logger.warning("no 'keystone_project_id' found while Updating container_chaining") - - @property - def authz_requests(self): - return self.__AUTHZ_REQUESTS - - # perimeter functions - - @property - def subjects(self): - return self.__SUBJECTS - - def __update_subjects(self, policy_id): - response = requests.get("{}/policies/{}/subjects".format(self.manager_url, policy_id)) - if 'subjects' in response.json(): - self.__SUBJECTS[policy_id] = response.json()['subjects'] - else: - raise exceptions.SubjectUnknown("Cannot find subject within policy_id {}".format(policy_id)) - - def get_subject(self, policy_id, name): - if not policy_id: - raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) - - if policy_id in self.subjects: - for _subject_id, _subject_dict in self.subjects[policy_id].items(): - if _subject_id == name or _subject_dict.get("name") == name: - return _subject_id - - self.__update_subjects(policy_id) - - if policy_id in self.subjects: - for _subject_id, _subject_dict in self.subjects[policy_id].items(): - if _subject_id == name or _subject_dict.get("name") == name: - return _subject_id - - raise exceptions.SubjectUnknown("Cannot find subject {}".format(name)) - - @property - def objects(self): - return self.__OBJECTS - - def __update_objects(self, policy_id): - response = requests.get("{}/policies/{}/objects".format(self.manager_url, policy_id)) - if 'objects' in response.json(): - self.__OBJECTS[policy_id] = response.json()['objects'] - else: - raise exceptions.ObjectUnknown("Cannot find object within policy_id {}".format(policy_id)) - - def get_object(self, policy_id, name): - if not policy_id: - raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) - - if policy_id in self.objects: - for _object_id, _object_dict in self.__OBJECTS[policy_id].items(): - if _object_id == name or _object_dict.get("name") == name: - return _object_id - - self.__update_objects(policy_id) - - if policy_id in self.objects: - for _object_id, _object_dict in self.__OBJECTS[policy_id].items(): - if _object_id == name or _object_dict.get("name") == name: - return _object_id - - raise exceptions.ObjectUnknown("Cannot find object {}".format(name)) - - @property - def actions(self): - return self.__ACTIONS - - def __update_actions(self, policy_id): - response = requests.get("{}/policies/{}/actions".format(self.manager_url, policy_id)) - - if 'actions' in response.json(): - self.__ACTIONS[policy_id] = response.json()['actions'] - else: - raise exceptions.ActionUnknown("Cannot find action within policy_id {}".format(policy_id)) - - def get_action(self, policy_id, name): - if not policy_id: - raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) - - if policy_id in self.actions: - for _action_id, _action_dict in self.__ACTIONS[policy_id].items(): - if _action_id == name or _action_dict.get("name") == name: - return _action_id - - self.__update_actions(policy_id) - - for _action_id, _action_dict in self.__ACTIONS[policy_id].items(): - if _action_id == name or _action_dict.get("name") == name: - return _action_id - - raise exceptions.ActionUnknown("Cannot find action {}".format(name)) - - # meta_rule functions - - @property - def meta_rules(self): - current_time = time.time() - if self.__META_RULES_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__META_RULES_UPDATE = current_time - self.__update_meta_rules() - self.__META_RULES_UPDATE = current_time - return self.__META_RULES - - def __update_meta_rules(self): - response = requests.get("{}/meta_rules".format(self.manager_url)) - - if 'meta_rules' in response.json(): - self.__META_RULES = response.json()['meta_rules'] - else: - raise exceptions.MetaRuleUnknown("Cannot find meta rules") - - # rule functions - - @property - def rules(self): - current_time = time.time() - if self.__RULES_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__RULES_UPDATE = current_time - self.__update_rules() - self.__RULES_UPDATE = current_time - return self.__RULES - - def __update_rules(self): - for policy_id in self.policies: - logger.debug("Get {}".format("{}/policies/{}/rules".format( - self.manager_url, policy_id))) - - response = requests.get("{}/policies/{}/rules".format( - self.manager_url, policy_id)) - if 'rules' in response.json(): - self.__RULES[policy_id] = response.json()['rules'] - else: - logger.warning(" no 'rules' found within policy_id: {}".format(policy_id)) - - logger.debug("UPDATE RULES {}".format(self.__RULES)) - - # assignment functions - - def update_assignments(self, policy_id=None, perimeter_id=None): - if policy_id: - self.__update_subject_assignments(policy_id=policy_id, perimeter_id=perimeter_id) - self.__update_object_assignments(policy_id=policy_id, perimeter_id=perimeter_id) - self.__update_action_assignments(policy_id=policy_id, perimeter_id=perimeter_id) - else: - for policy_id in self.__POLICIES: - self.__update_subject_assignments(policy_id=policy_id, perimeter_id=perimeter_id) - self.__update_object_assignments(policy_id=policy_id, perimeter_id=perimeter_id) - self.__update_action_assignments(policy_id=policy_id, perimeter_id=perimeter_id) - - @property - def subject_assignments(self): - return self.__SUBJECT_ASSIGNMENTS - - def __update_subject_assignments(self, policy_id, perimeter_id=None): - if perimeter_id: - response = requests.get("{}/policies/{}/subject_assignments/{}".format( - self.manager_url, policy_id, perimeter_id)) - else: - response = requests.get("{}/policies/{}/subject_assignments".format( - self.manager_url, policy_id)) - - if 'subject_assignments' in response.json(): - if policy_id not in self.subject_assignments: - self.__SUBJECT_ASSIGNMENTS[policy_id] = {} - self.__SUBJECT_ASSIGNMENTS[policy_id] = response.json()['subject_assignments'] - else: - raise exceptions.SubjectAssignmentUnknown( - "Cannot find subject assignment within policy_id {}".format(policy_id)) - - def get_subject_assignments(self, policy_id, perimeter_id, category_id): - if not policy_id: - raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) - - if policy_id not in self.subject_assignments: - self.__update_subject_assignments(policy_id, perimeter_id) - - for key, value in self.subject_assignments[policy_id].items(): - if all(k in value for k in ("subject_id", "category_id", "assignments")): - if perimeter_id == value['subject_id'] and category_id == value['category_id']: - return value['assignments'] - else: - logger.warning("'subject_id' or 'category_id' or 'assignments'" - " keys are not found in subject_assignments") - return [] - - @property - def object_assignments(self): - return self.__OBJECT_ASSIGNMENTS - - def __update_object_assignments(self, policy_id, perimeter_id=None): - if perimeter_id: - response = requests.get("{}/policies/{}/object_assignments/{}".format( - self.manager_url, policy_id, perimeter_id)) - else: - response = requests.get("{}/policies/{}/object_assignments".format( - self.manager_url, policy_id)) - - if 'object_assignments' in response.json(): - if policy_id not in self.object_assignments: - self.__OBJECT_ASSIGNMENTS[policy_id] = {} - - self.__OBJECT_ASSIGNMENTS[policy_id] = response.json()['object_assignments'] - else: - raise exceptions.ObjectAssignmentUnknown( - "Cannot find object assignment within policy_id {}".format(policy_id)) - - def get_object_assignments(self, policy_id, perimeter_id, category_id): - if not policy_id: - raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) - - if policy_id not in self.object_assignments: - self.__update_object_assignments(policy_id, perimeter_id) - - for key, value in self.object_assignments[policy_id].items(): - if all(k in value for k in ("object_id", "category_id", "assignments")): - if perimeter_id == value['object_id'] and category_id == value['category_id']: - return value['assignments'] - else: - logger.warning("'object_id' or 'category_id' or'assignments'" - " keys are not found in object_assignments") - return [] - - @property - def action_assignments(self): - return self.__ACTION_ASSIGNMENTS - - def __update_action_assignments(self, policy_id, perimeter_id=None): - if perimeter_id: - response = requests.get("{}/policies/{}/action_assignments/{}".format( - self.manager_url, policy_id, perimeter_id)) - else: - response = requests.get("{}/policies/{}/action_assignments".format( - self.manager_url, policy_id)) - - if 'action_assignments' in response.json(): - if policy_id not in self.__ACTION_ASSIGNMENTS: - self.__ACTION_ASSIGNMENTS[policy_id] = {} - - self.__ACTION_ASSIGNMENTS[policy_id] = response.json()['action_assignments'] - else: - raise exceptions.ActionAssignmentUnknown( - "Cannot find action assignment within policy_id {}".format(policy_id)) - - def get_action_assignments(self, policy_id, perimeter_id, category_id): - if not policy_id: - raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) - - if policy_id not in self.action_assignments: - self.__update_action_assignments(policy_id, perimeter_id) - - for key, value in self.action_assignments[policy_id].items(): - if all(k in value for k in ("action_id", "category_id", "assignments")): - if perimeter_id == value['action_id'] and category_id == value['category_id']: - return value['assignments'] - else: - logger.warning("'action_id' or 'category_id' or'assignments'" - " keys are not found in action_assignments") - return [] - - # category functions - - @property - def subject_categories(self): - current_time = time.time() - if self.__SUBJECT_CATEGORIES_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__SUBJECT_CATEGORIES_UPDATE = current_time - self.__update_subject_categories() - self.__SUBJECT_CATEGORIES_UPDATE = current_time - return self.__SUBJECT_CATEGORIES - - def __update_subject_categories(self): - response = requests.get("{}/policies/subject_categories".format( - self.manager_url)) - - if 'subject_categories' in response.json(): - self.__SUBJECT_CATEGORIES.update(response.json()['subject_categories']) - else: - raise exceptions.SubjectCategoryUnknown("Cannot find subject category") - - @property - def object_categories(self): - current_time = time.time() - if self.__OBJECT_CATEGORIES_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__OBJECT_CATEGORIES_UPDATE = current_time - self.__update_object_categories() - self.__OBJECT_CATEGORIES_UPDATE = current_time - return self.__OBJECT_CATEGORIES - - def __update_object_categories(self): - response = requests.get("{}/policies/object_categories".format(self.manager_url)) - - if 'object_categories' in response.json(): - self.__OBJECT_CATEGORIES.update(response.json()['object_categories']) - else: - raise exceptions.ObjectCategoryUnknown("Cannot find object category") - - @property - def action_categories(self): - current_time = time.time() - if self.__ACTION_CATEGORIES_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__ACTION_CATEGORIES_UPDATE = current_time - self.__update_action_categories() - self.__ACTION_CATEGORIES_UPDATE = current_time - return self.__ACTION_CATEGORIES - - def __update_action_categories(self): - response = requests.get("{}/policies/action_categories".format(self.manager_url)) - - if 'action_categories' in response.json(): - self.__ACTION_CATEGORIES.update(response.json()['action_categories']) - else: - raise exceptions.ActionCategoryUnknown("Cannot find action category") - - # PDP functions - - def __update_pdp(self): - response = requests.get("{}/pdp".format(self.manager_url)) - pdp = response.json() - if 'pdps' in pdp: - for _pdp in pdp["pdps"].values(): - if "keystone_project_id" in _pdp and _pdp['keystone_project_id'] not in self.container_chaining: - self.__CONTAINER_CHAINING[_pdp['keystone_project_id']] = {} - # Note (asteroide): force update of chaining - self.__update_container_chaining(_pdp['keystone_project_id']) - for key, value in pdp["pdps"].items(): - self.__PDP[key] = value - - else: - raise exceptions.PdpError("Cannot find 'pdps' key") - - @property - def pdp(self): - """Policy Decision Point - Example of content: - { - "pdp_id": { - "keystone_project_id": "keystone_project_id", - "name": "pdp1", - "description": "test", - "security_pipeline": [ - "policy_id" - ] - } - } - - :return: - """ - current_time = time.time() - if self.__PDP_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__PDP_UPDATE = current_time - self.__update_pdp() - self.__PDP_UPDATE = current_time - return self.__PDP - - # policy functions - def __update_policies(self): - response = requests.get("{}/policies".format(self.manager_url)) - policies = response.json() - - if 'policies' in policies: - for key, value in policies["policies"].items(): - self.__POLICIES[key] = value - else: - raise exceptions.PolicytNotFound("Cannot find 'policies' key") - - @property - def policies(self): - current_time = time.time() - if self.__POLICIES_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__POLICIES_UPDATE = current_time - self.__update_policies() - self.__POLICIES_UPDATE = current_time - return self.__POLICIES - - # model functions - - def __update_models(self): - response = requests.get("{}/models".format(self.manager_url)) - models = response.json() - if 'models' in models: - for key, value in models["models"].items(): - self.__MODELS[key] = value - else: - raise exceptions.ModelNotFound("Cannot find 'models' key") - - @property - def models(self): - current_time = time.time() - if self.__MODELS_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__MODELS_UPDATE = current_time - self.__update_models() - self.__MODELS_UPDATE = current_time - return self.__MODELS - - # helper functions - - def get_policy_from_meta_rules(self, meta_rule_id): - for pdp_key, pdp_value in self.pdp.items(): - if "security_pipeline" in pdp_value: - for policy_id in pdp_value["security_pipeline"]: - if policy_id in self.policies and "model_id" in self.policies[policy_id]: - model_id = self.policies[policy_id]["model_id"] - if model_id in self.models and "meta_rules" in self.models[model_id]: - if meta_rule_id in self.models[model_id]["meta_rules"]: - return policy_id - else: - logger.warning( - "Cannot find model_id: {} within " - "models and 'meta_rules' key".format(model_id)) - else: - logger.warning( - "Cannot find policy_id: {} " - "within policies and 'model_id' key".format( - policy_id)) - else: - logger.warning("Cannot find 'security_pipeline' " - "key within pdp ") - - def get_meta_rule_ids_from_pdp_value(self, pdp_value): - meta_rules = [] - if "security_pipeline" in pdp_value: - for policy_id in pdp_value["security_pipeline"]: - if policy_id not in self.policies or "model_id" not in self.policies[policy_id]: - raise exceptions.PolicyUnknown("Cannot find 'models' key") - model_id = self.policies[policy_id]["model_id"] - if model_id not in self.models or 'meta_rules' not in self.models[model_id]: - raise exceptions.ModelNotFound("Cannot find 'models' key") - for meta_rule in self.models[model_id]["meta_rules"]: - meta_rules.append(meta_rule) - return meta_rules - raise exceptions.PdpContentError - - def get_pdp_from_keystone_project(self, keystone_project_id): - for pdp_key, pdp_value in self.pdp.items(): - if "keystone_project_id" in pdp_value and \ - keystone_project_id == pdp_value["keystone_project_id"]: - return pdp_key - - def get_keystone_project_id_from_policy_id(self, policy_id): - for pdp_key, pdp_value in self.pdp.items(): - if "security_pipeline" in pdp_value and \ - "keystone_project_id" in pdp_value: - if policy_id in pdp_value["security_pipeline"]: - return pdp_value["keystone_project_id"] - else: - logger.warning(" 'security_pipeline','keystone_project_id' " - "key not in pdp {}".format(pdp_value)) - - def get_keystone_project_id_from_pdp_id(self, pdp_id): - if pdp_id in self.pdp: - pdp_value = self.pdp.get(pdp_id) - if "security_pipeline" in pdp_value and \ - "keystone_project_id" in pdp_value: - return pdp_value["keystone_project_id"] - logger.warning("Unknown PDP ID".format(pdp_id)) - - def get_containers_from_keystone_project_id(self, keystone_project_id, - meta_rule_id=None): - for container_id, container_values in self.containers.items(): - for container_value in container_values: - if 'keystone_project_id' not in container_value: - continue - if container_value['keystone_project_id'] == keystone_project_id: - if not meta_rule_id: - yield container_id, container_value - elif "meta_rule_id" in container_value and \ - container_value.get('meta_rule_id') == meta_rule_id: - yield container_id, container_value - break - - # containers functions - - def __update_container(self): - response = requests.get("{}/pods".format(self.orchestrator_url)) - pods = response.json() - if "pods" in pods: - for key, value in pods["pods"].items(): - # if key not in self.__CONTAINERS: - self.__CONTAINERS[key] = value - # else: - # for container in value: - # self.__CONTAINERS[key].update(value) - else: - raise exceptions.PodError("Cannot find 'pods' key") - - def add_container(self, container_data): - """Add a new container in the cache - - :param container_data: dictionary with information for the container - Example: - { - "name": name, - "hostname": name, - "port": { - "PrivatePort": tcp_port, - "Type": "tcp", - "IP": "0.0.0.0", - "PublicPort": tcp_port - }, - "keystone_project_id": uuid, - "pdp_id": self.CACHE.get_pdp_from_keystone_project(uuid), - "meta_rule_id": meta_rule_id, - "container_name": container_name, - "plugin_name": plugin_name - "container_id": "container_id" - } - - :return: - """ - if all(k in container_data for k in ("keystone_project_id", "name", "container_id", "policy_id", - "meta_rule_id", "port")) \ - and all(k in container_data['port'] for k in ("PublicPort", "Type", "IP", "PrivatePort")): - - self.__CONTAINERS[uuid4().hex] = { - "keystone_project_id": container_data['keystone_project_id'], - "name": container_data['name'], - "container_id": container_data['container_id'], - "hostname": container_data['name'], - "policy_id": container_data['policy_id'], - "meta_rule_id": container_data['meta_rule_id'], - "port": [ - { - "PublicPort": container_data['port']["PublicPort"], - "Type": container_data['port']["Type"], - "IP": container_data['port']["IP"], - "PrivatePort": container_data['port']["PrivatePort"] - } - ], - "genre": container_data['plugin_name'] - } - self.__update_container_chaining(self.get_keystone_project_id_from_policy_id(container_data['policy_id'])) - else: - raise exceptions.ContainerError("Cannot find 'container' parameters key") - - @property - def containers(self): - """Containers cache - example of content : - { - "policy_uuid1": "component_hostname1", - "policy_uuid2": "component_hostname2", - } - :return: - """ - current_time = time.time() - if self.__CONTAINERS_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__CONTAINERS_UPDATE = current_time - self.__update_container() - self.__CONTAINERS_UPDATE = current_time - return self.__CONTAINERS - - @property - def container_chaining(self): - """Cache for mapping Keystone Project ID with meta_rule ID and container ID - Example of content: - { - "keystone_project_id": [ - { - "container_id": "container_id", - "genre": "genre", - "policy_id": "policy_id", - "meta_rule_id": "meta_rule_id", - } - ] - } - - :return: - """ - current_time = time.time() - if self.__CONTAINER_CHAINING_UPDATE + self.__UPDATE_INTERVAL < current_time: - self.__CONTAINER_CHAINING_UPDATE = current_time - for key, value in self.pdp.items(): - if "keystone_project_id" in value: - if not value["keystone_project_id"]: - continue - self.__update_container_chaining(value["keystone_project_id"]) - else: - logger.warning("no 'keystone_project_id' found") - self.__CONTAINER_CHAINING_UPDATE = current_time - return self.__CONTAINER_CHAINING - - def __update_container_chaining(self, keystone_project_id): - container_ids = [] - for pdp_id, pdp_value, in self.__PDP.items(): - if pdp_value: - if all(k in pdp_value for k in ("keystone_project_id", "security_pipeline")) \ - and pdp_value["keystone_project_id"] == keystone_project_id: - for policy_id in pdp_value["security_pipeline"]: - if policy_id in self.policies and "model_id" in self.policies[policy_id]: - model_id = self.policies[policy_id]['model_id'] - if model_id in self.models and "meta_rules" in self.models[model_id]: - for meta_rule_id in self.models[model_id]["meta_rules"]: - for container_id, container_value in self.get_containers_from_keystone_project_id( - keystone_project_id, - meta_rule_id - ): - if "name" in container_value: - if all(k in container_value for k in ("genre", "port")): - container_ids.append( - { - "container_id": container_value["name"], - "genre": container_value["genre"], - "policy_id": policy_id, - "meta_rule_id": meta_rule_id, - "hostname": container_value["name"], - "hostip": "127.0.0.1", - "port": container_value["port"], - } - ) - else: - logger.warning("Container content keys not found {}", container_value) - else: - logger.warning("Container content keys not found {}", container_value) - else: - raise exceptions.ModelUnknown("Cannot find model_id: {} in models and " - "may not contains 'meta_rules' key".format(model_id)) - else: - raise exceptions.PolicyUnknown("Cannot find policy within policy_id: {}, " - "and may not contains 'model_id' key".format(policy_id)) - - self.__CONTAINER_CHAINING[keystone_project_id] = container_ids |