diff options
Diffstat (limited to 'moon_manager/moon_manager/api/json_import.py')
| -rw-r--r-- | moon_manager/moon_manager/api/json_import.py | 186 |
1 files changed, 123 insertions, 63 deletions
diff --git a/moon_manager/moon_manager/api/json_import.py b/moon_manager/moon_manager/api/json_import.py index e57a27c1..05f4a0c0 100644 --- a/moon_manager/moon_manager/api/json_import.py +++ b/moon_manager/moon_manager/api/json_import.py @@ -19,7 +19,6 @@ from python_moondb.core import PDPManager from python_moondb.core import PolicyManager from python_moondb.core import ModelManager - __version__ = "4.5.0" logger = logging.getLogger("moon.manager.api." + __name__) @@ -32,64 +31,61 @@ CATEGORIES_CALLBACK = 3 class ForbiddenOverride(BaseException): def __init__(self, message): - # Call the base class constructor with the parameters it needs super(ForbiddenOverride, self).__init__(message) class UnknownPolicy(BaseException): def __init__(self, message): - # Call the base class constructor with the parameters it needs super(UnknownPolicy, self).__init__(message) class UnknownModel(BaseException): def __init__(self, message): - # Call the base class constructor with the parameters it needs super(UnknownModel, self).__init__(message) class UnknownData(BaseException): def __init__(self, message): - # Call the base class constructor with the parameters it needs super(UnknownData, self).__init__(message) class MissingPolicy(BaseException): def __init__(self, message): - # Call the base class constructor with the parameters it needs super(MissingPolicy, self).__init__(message) class InvalidJson(BaseException): def __init__(self, message): - # Call the base class constructor with the parameters it needs super(InvalidJson, self).__init__(message) class JsonImport(Resource): - __urls__ = ( "/import", "/import/", ) - def _reorder_rules_ids(self, rule, ordered_perimeter_categories_ids, json_data_ids, policy_id, get_function): - ordered_json_ids = [None]*len(ordered_perimeter_categories_ids) + def _reorder_rules_ids(self, rule, ordered_perimeter_categories_ids, json_data_ids, policy_id, + get_function): + ordered_json_ids = [None] * len(ordered_perimeter_categories_ids) for json_id in json_data_ids: data = get_function(self._user_id, policy_id, data_id=json_id) data = data[0] if data["category_id"] not in ordered_perimeter_categories_ids: - raise InvalidJson("The category id {} of the rule {} does not match the meta rule".format( - data["category_id"], rule)) - if ordered_json_ids[ordered_perimeter_categories_ids.index(data["category_id"])] is not None: - raise InvalidJson("The category id {} of the rule {} shall not be used twice in the same rule".format( - data["category_id"], rule)) + raise InvalidJson( + "The category id {} of the rule {} does not match the meta rule".format( + data["category_id"], rule)) + if ordered_json_ids[ + ordered_perimeter_categories_ids.index(data["category_id"])] is not None: + raise InvalidJson( + "The category id {} of the rule {} shall not be used twice in the same rule".format( + data["category_id"], rule)) ordered_json_ids[ordered_perimeter_categories_ids.index(data["category_id"])] = json_id logger.info(ordered_json_ids) return ordered_json_ids @@ -101,30 +97,46 @@ class JsonImport(Resource): for json_rule in json_rules: json_to_use = dict() JsonUtils.copy_field_if_exists(json_rule, json_to_use, "instructions", str) - JsonUtils.copy_field_if_exists(json_rule, json_to_use, "enabled", bool, default_value=True) + JsonUtils.copy_field_if_exists(json_rule, json_to_use, "enabled", bool, + default_value=True) json_ids = dict() JsonUtils.convert_name_to_id(json_rule, json_ids, "policy", "policy_id", "policy", PolicyManager, self._user_id) - JsonUtils.convert_name_to_id(json_rule, json_to_use, "meta_rule", "meta_rule_id", "meta_rule", ModelManager, self._user_id) + JsonUtils.convert_name_to_id(json_rule, json_to_use, "meta_rule", "meta_rule_id", + "meta_rule", ModelManager, self._user_id) json_subject_ids = dict() json_object_ids = dict() json_action_ids = dict() - JsonUtils.convert_names_to_ids(json_rule["rule"], json_subject_ids, "subject_data", "subject", "subject_data", PolicyManager, self._user_id, json_ids["policy_id"]) - JsonUtils.convert_names_to_ids(json_rule["rule"], json_object_ids, "object_data", "object", "object_data", PolicyManager, self._user_id, json_ids["policy_id"]) - JsonUtils.convert_names_to_ids(json_rule["rule"], json_action_ids, "action_data", "action", "action_data", PolicyManager, self._user_id, json_ids["policy_id"]) + JsonUtils.convert_names_to_ids(json_rule["rule"], json_subject_ids, "subject_data", + "subject", "subject_data", PolicyManager, self._user_id, + json_ids["policy_id"]) + JsonUtils.convert_names_to_ids(json_rule["rule"], json_object_ids, "object_data", + "object", "object_data", PolicyManager, self._user_id, + json_ids["policy_id"]) + JsonUtils.convert_names_to_ids(json_rule["rule"], json_action_ids, "action_data", + "action", "action_data", PolicyManager, self._user_id, + json_ids["policy_id"]) meta_rule = ModelManager.get_meta_rules(self._user_id, json_to_use["meta_rule_id"]) meta_rule = [v for v in meta_rule.values()] meta_rule = meta_rule[0] - json_to_use_rule = self._reorder_rules_ids(json_rule, meta_rule["subject_categories"], json_subject_ids["subject"], json_ids["policy_id"], PolicyManager.get_subject_data) - json_to_use_rule = json_to_use_rule + self._reorder_rules_ids(json_rule, meta_rule["object_categories"], json_object_ids["object"], json_ids["policy_id"], PolicyManager.get_object_data) - json_to_use_rule = json_to_use_rule + self._reorder_rules_ids(json_rule, meta_rule["action_categories"], json_action_ids["action"], json_ids["policy_id"], PolicyManager.get_action_data) + json_to_use_rule = self._reorder_rules_ids(json_rule, meta_rule["subject_categories"], + json_subject_ids["subject"], + json_ids["policy_id"], + PolicyManager.get_subject_data) + json_to_use_rule = json_to_use_rule + self._reorder_rules_ids(json_rule, meta_rule[ + "object_categories"], json_object_ids["object"], json_ids["policy_id"], + PolicyManager.get_object_data) + json_to_use_rule = json_to_use_rule + self._reorder_rules_ids(json_rule, meta_rule[ + "action_categories"], json_action_ids["action"], json_ids["policy_id"], + PolicyManager.get_action_data) json_to_use["rule"] = json_to_use_rule try: logger.debug("Adding / updating a rule from json {}".format(json_to_use)) - PolicyManager.add_rule(self._user_id, json_ids["policy_id"], json_to_use["meta_rule_id"], json_to_use) + PolicyManager.add_rule(self._user_id, json_ids["policy_id"], + json_to_use["meta_rule_id"], json_to_use) except exceptions.RuleExisting: pass except exceptions.PolicyUnknown: @@ -136,11 +148,18 @@ class JsonImport(Resource): json_to_use = dict() JsonUtils.copy_field_if_exists(json_meta_rule, json_to_use, "name", str) JsonUtils.copy_field_if_exists(json_meta_rule, json_to_use, "description", str) - JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "subject_categories", "subject_categories", "subject_category", ModelManager, self._user_id) - JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "object_categories", "object_categories", "object_category", ModelManager, self._user_id) - JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "action_categories", "action_categories", "action_category", ModelManager, self._user_id) + JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "subject_categories", + "subject_categories", "subject_category", ModelManager, + self._user_id) + JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "object_categories", + "object_categories", "object_category", ModelManager, + self._user_id) + JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "action_categories", + "action_categories", "action_category", ModelManager, + self._user_id) logger.debug("Adding / updating a metarule from json {}".format(json_meta_rule)) - meta_rule = ModelManager.add_meta_rule(self._user_id, meta_rule_id=None, value=json_to_use) + meta_rule = ModelManager.add_meta_rule(self._user_id, meta_rule_id=None, + value=json_to_use) logger.debug("Added / updated meta rule : {}".format(meta_rule)) def _import_subject_object_action_assignments(self, json_item_assignments, type_element): @@ -156,29 +175,40 @@ class JsonImport(Resource): for json_item_assignment in json_item_assignments: item_override = JsonUtils.get_override(json_item_assignment) if item_override is True: - raise ForbiddenOverride("{} assignments do not support override flag !".format(type_element)) + raise ForbiddenOverride( + "{} assignments do not support override flag !".format(type_element)) json_assignment = dict() - JsonUtils.convert_name_to_id(json_item_assignment, json_assignment, "category", "category_id", type_element + "_category", ModelManager, self._user_id) + JsonUtils.convert_name_to_id(json_item_assignment, json_assignment, "category", + "category_id", type_element + "_category", ModelManager, + self._user_id) has_found_data = False # loop over policies for policy_id in policies: json_data = dict() try: - JsonUtils.convert_name_to_id(json_item_assignment, json_assignment, type_element, "id", type_element, PolicyManager, self._user_id, policy_id) - JsonUtils.convert_names_to_ids(json_item_assignment, json_data, "assignments", "data_id", type_element + "_data", PolicyManager, self._user_id, policy_id, json_assignment["category_id"]) + JsonUtils.convert_name_to_id(json_item_assignment, json_assignment, + type_element, "id", type_element, PolicyManager, + self._user_id, policy_id) + JsonUtils.convert_names_to_ids(json_item_assignment, json_data, "assignments", + "data_id", type_element + "_data", PolicyManager, + self._user_id, policy_id, + json_assignment["category_id"]) has_found_data = True except UnknownName: # the category or data has not been found in this policy : we look into the next one continue for data_id in json_data["data_id"]: # find the policy related to the current data - data = get_method(self._user_id, policy_id, data_id, json_assignment["category_id"]) + data = get_method(self._user_id, policy_id, data_id, + json_assignment["category_id"]) if data is not None and len(data) == 1: - logger.debug("Adding / updating a {} assignment from json {}".format(type_element, - json_assignment)) - import_method(self._user_id, policy_id, json_assignment["id"], json_assignment["category_id"], + logger.debug( + "Adding / updating a {} assignment from json {}".format(type_element, + json_assignment)) + import_method(self._user_id, policy_id, json_assignment["id"], + json_assignment["category_id"], data_id) else: raise UnknownData("Unknown data with id {}".format(data_id)) @@ -189,7 +219,8 @@ class JsonImport(Resource): type_element, json_item_assignment)) - def _import_subject_object_action_datas(self, json_items_data, mandatory_policy_ids, type_element): + def _import_subject_object_action_datas(self, json_items_data, mandatory_policy_ids, + type_element): if type_element == "subject": import_method = getattr(PolicyManager, 'set_' + type_element + '_data') else: @@ -202,16 +233,20 @@ class JsonImport(Resource): for json_item_data in json_items_data: item_override = JsonUtils.get_override(json_items_data) if item_override is True: - raise ForbiddenOverride("{} datas do not support override flag !".format(type_element)) + raise ForbiddenOverride( + "{} datas do not support override flag !".format(type_element)) json_to_use = dict() JsonUtils.copy_field_if_exists(json_item_data, json_to_use, "name", str) JsonUtils.copy_field_if_exists(json_item_data, json_to_use, "description", str) json_policy = dict() # field_mandatory : not mandatory if there is some mandatory policies - JsonUtils.convert_names_to_ids(json_item_data, json_policy, "policies", "policy_id", "policy", - PolicyManager, self._user_id, field_mandatory=len(mandatory_policy_ids) == 0) + JsonUtils.convert_names_to_ids(json_item_data, json_policy, "policies", "policy_id", + "policy", + PolicyManager, self._user_id, + field_mandatory=len(mandatory_policy_ids) == 0) json_category = dict() - JsonUtils.convert_name_to_id(json_item_data, json_category, "category", "category_id", type_element+"_category", + JsonUtils.convert_name_to_id(json_item_data, json_category, "category", "category_id", + type_element + "_category", ModelManager, self._user_id) policy_ids = [] if "policy_id" in json_policy: @@ -222,16 +257,20 @@ class JsonImport(Resource): mandatory_policy_ids.append(policy_id) if len(mandatory_policy_ids) == 0: - raise InvalidJson("Invalid data, the policy shall be set when importing {}".format(json_item_data)) + raise InvalidJson("Invalid data, the policy shall be set when importing {}".format( + json_item_data)) category_id = None if "category_id" in json_category: category_id = json_category["category_id"] if category_id is None: - raise InvalidJson("Invalid data, the category shall be set when importing {}".format(json_item_data)) + raise InvalidJson( + "Invalid data, the category shall be set when importing {}".format( + json_item_data)) for policy_id in mandatory_policy_ids: try: - data = import_method(self._user_id, policy_id, category_id=category_id, value=json_to_use) + data = import_method(self._user_id, policy_id, category_id=category_id, + value=json_to_use) except exceptions.PolicyUnknown: raise UnknownPolicy("Unknown policy with id {}".format(policy_id)) except Exception as e: @@ -260,13 +299,16 @@ class JsonImport(Resource): JsonUtils.copy_field_if_exists(json_item_category, json_to_use, "description", str) item_override = JsonUtils.get_override(json_item_category) if item_override is True: - raise ForbiddenOverride("{} categories do not support override flag !".format(type_element)) + raise ForbiddenOverride( + "{} categories do not support override flag !".format(type_element)) try: category = import_method(self._user_id, existing_id, json_to_use) - except (exceptions.SubjectCategoryExisting, exceptions.ObjectCategoryExisting, exceptions.ActionCategoryExisting): + except (exceptions.SubjectCategoryExisting, exceptions.ObjectCategoryExisting, + exceptions.ActionCategoryExisting): # it already exists: do nothing - logger.warning("Ignored {} category with name {} is already in the database".format(type_element, json_to_use["name"])) + logger.warning("Ignored {} category with name {} is already in the database".format( + type_element, json_to_use["name"])) except Exception as e: logger.warning("Error while importing the category : {}".format(str(e))) logger.exception(str(e)) @@ -284,7 +326,9 @@ class JsonImport(Resource): JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "name", str) JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "description", str) JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "extra", dict) - JsonUtils.convert_names_to_ids(json_item, json_without_policy_name, "policies", "policy_list", "policy", PolicyManager, self._user_id, field_mandatory=False) + JsonUtils.convert_names_to_ids(json_item, json_without_policy_name, "policies", + "policy_list", "policy", PolicyManager, self._user_id, + field_mandatory=False) policy_ids = json_without_policy_name["policy_list"] for mandatory_policy_id in mandatory_policy_ids: if mandatory_policy_id not in policy_ids: @@ -297,7 +341,9 @@ class JsonImport(Resource): raise ForbiddenOverride("{} does not support override flag !".format(type_element)) if len(policy_ids) == 0: - raise MissingPolicy("a {} needs at least one policy to be created or updated : {}".format(type_element, json.dumps(json_item))) + raise MissingPolicy( + "a {} needs at least one policy to be created or updated : {}".format( + type_element, json.dumps(json_item))) for policy_id in policy_ids: try: @@ -307,7 +353,8 @@ class JsonImport(Resource): if items_in_db[key_in_db]["name"] == json_without_policy_name["name"]: key = key_in_db break - element = import_method(self._user_id, policy_id, perimeter_id=key, value=json_without_policy_name) + element = import_method(self._user_id, policy_id, perimeter_id=key, + value=json_without_policy_name) logger.debug("Added / updated {} : {}".format(type_element, element)) except exceptions.PolicyUnknown: @@ -344,24 +391,29 @@ class JsonImport(Resource): if policy_override is False and policy_does_exist: if policy_id: policy_mandatory_ids.append(policy_id) - logger.warning("Existing policy not updated because of the override option is not set !") + logger.warning( + "Existing policy not updated because of the override option is not set !") continue json_without_model_name = dict() JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "name", str) JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "description", str) JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "genre", str) - JsonUtils.convert_name_to_id(json_policy, json_without_model_name, "model", "model_id", "model", ModelManager, self._user_id, field_mandatory=False) + JsonUtils.convert_name_to_id(json_policy, json_without_model_name, "model", "model_id", + "model", ModelManager, self._user_id, + field_mandatory=False) if not policy_does_exist: logger.debug("Creating policy {} ".format(json_without_model_name)) - added_policy = PolicyManager.add_policy(self._user_id, None, json_without_model_name) + added_policy = PolicyManager.add_policy(self._user_id, None, + json_without_model_name) if policy_mandatory is True: keys = list(added_policy.keys()) policy_mandatory_ids.append(keys[0]) elif policy_override is True: logger.debug("Updating policy {} ".format(json_without_model_name)) - updated_policy = PolicyManager.update_policy(self._user_id, policy_id, json_without_model_name) + updated_policy = PolicyManager.update_policy(self._user_id, policy_id, + json_without_model_name) if policy_mandatory is True: policy_mandatory_ids.append(policy_id) return policy_mandatory_ids @@ -376,7 +428,8 @@ class JsonImport(Resource): model_in_db = None model_id = None for model_key in models: - if ("id" in json_model and model_key == json_model["id"]) or ("name" in json_model and models[model_key]["name"] == json_model["name"]): + if ("id" in json_model and model_key == json_model["id"]) or ( + "name" in json_model and models[model_key]["name"] == json_model["name"]): model_in_db = models[model_key] model_id = model_key @@ -385,7 +438,8 @@ class JsonImport(Resource): raise UnknownModel("Unknown model ") json_key = dict() - JsonUtils.convert_names_to_ids(json_model, json_key, "meta_rules", "meta_rule_id", "meta_rule", ModelManager, self._user_id) + JsonUtils.convert_names_to_ids(json_model, json_key, "meta_rules", "meta_rule_id", + "meta_rule", ModelManager, self._user_id) for meta_rule_id in json_key["meta_rule_id"]: if meta_rule_id not in model_in_db["meta_rules"]: model_in_db["meta_rules"].append(meta_rule_id) @@ -410,18 +464,20 @@ class JsonImport(Resource): model_id = model_key # end TODO - JsonUtils.copy_field_if_exists(json_model, json_without_new_metarules, "description", str) + JsonUtils.copy_field_if_exists(json_model, json_without_new_metarules, "description", + str) if model_in_db is None: model_does_exist = False else: - json_without_new_metarules["meta_rule_id"] = model_in_db["meta_rules"] + json_without_new_metarules["meta_rules"] = model_in_db["meta_rules"] model_does_exist = True model_override = JsonUtils.get_override(json_model) if not model_does_exist: logger.debug("Creating model {} ".format(json_without_new_metarules)) ModelManager.add_model(self._user_id, None, json_without_new_metarules) elif model_override is True: - logger.debug("Updating model with id {} : {} ".format(model_id, json_without_new_metarules)) + logger.debug( + "Updating model with id {} : {} ".format(model_id, json_without_new_metarules)) ModelManager.update_model(self._user_id, model_id, json_without_new_metarules) def _import_pdps(self, json_pdps): @@ -477,10 +533,6 @@ class JsonImport(Resource): if key in json_content: logger.info("Importing {}...".format(key)) self._import_subject_object_action_categories(json_content[key], in_key) - key = in_key + "_data" - if key in json_content: - logger.info("Importing {}...".format(key)) - self._import_subject_object_action_datas(json_content[key], mandatory_policy_ids, in_key) # import meta rules if "meta_rules" in json_content: @@ -492,6 +544,14 @@ class JsonImport(Resource): logger.info("Updating models with meta rules...") self._import_models_with_new_meta_rules(json_content["models"]) + for elt in list_element: + in_key = elt["key"] + key = in_key + "_data" + if key in json_content: + logger.info("Importing {}...".format(key)) + self._import_subject_object_action_datas(json_content[key], mandatory_policy_ids, + in_key) + # import subjects assignments, idem for object and action for elt in list_element: in_key = elt["key"] |