From 7bb53c64da2dcf88894bfd31503accdd81498f3d Mon Sep 17 00:00:00 2001 From: Thomas Duval Date: Wed, 3 Jun 2020 10:06:52 +0200 Subject: Update to new version 5.4 Signed-off-by: Thomas Duval Change-Id: Idcd868133d75928a1ffd74d749ce98503e0555ea --- moon_manager/moon_manager/api/json_import.py | 600 ++------------------------- 1 file changed, 30 insertions(+), 570 deletions(-) (limited to 'moon_manager/moon_manager/api/json_import.py') diff --git a/moon_manager/moon_manager/api/json_import.py b/moon_manager/moon_manager/api/json_import.py index 05f4a0c0..922c5c2c 100644 --- a/moon_manager/moon_manager/api/json_import.py +++ b/moon_manager/moon_manager/api/json_import.py @@ -1,27 +1,23 @@ -# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors -# This software is distributed under the terms and conditions of the 'Apache-2.0' -# license which can be found in the file 'LICENSE' in this package distribution -# or at 'http://www.apache.org/licenses/LICENSE-2.0'. +# Software Name: MOON -from flask import request -from flask_restful import Resource -import flask_restful -from flask import abort +# Version: 5.4 + +# SPDX-FileCopyrightText: Copyright (c) 2018-2020 Orange and its contributors +# SPDX-License-Identifier: Apache-2.0 + +# This software is distributed under the 'Apache License 2.0', +# the text of which is available at 'http://www.apache.org/licenses/LICENSE-2.0.txt' +# or see the "LICENSE" file for more details. -from python_moonutilities.security_functions import check_auth -from python_moonutilities import exceptions -import logging -import json -from moon_manager.api.base_exception import BaseException -from moon_manager.api.json_utils import JsonUtils, UnknownName -from python_moondb.core import PDPManager -from python_moondb.core import PolicyManager -from python_moondb.core import ModelManager +import hug +import logging +from moon_manager import db_driver as driver +from moon_utilities.json_utils import JsonImport +from moon_utilities.auth_functions import api_key_authentication -__version__ = "4.5.0" -logger = logging.getLogger("moon.manager.api." + __name__) +LOGGER = logging.getLogger("moon.manager.api." + __name__) INST_CALLBACK = 0 DATA_CALLBACK = 1 @@ -29,556 +25,20 @@ ASSIGNMENT_CALLBACK = 2 CATEGORIES_CALLBACK = 3 -class ForbiddenOverride(BaseException): - def __init__(self, message): - # Call the base class constructor with the parameters it needs - super(ForbiddenOverride, self).__init__(message) - - -class UnknownPolicy(BaseException): - def __init__(self, message): - # Call the base class constructor with the parameters it needs - super(UnknownPolicy, self).__init__(message) - - -class UnknownModel(BaseException): - def __init__(self, message): - # Call the base class constructor with the parameters it needs - super(UnknownModel, self).__init__(message) - - -class UnknownData(BaseException): - def __init__(self, message): - # Call the base class constructor with the parameters it needs - super(UnknownData, self).__init__(message) - - -class MissingPolicy(BaseException): - def __init__(self, message): - # Call the base class constructor with the parameters it needs - super(MissingPolicy, self).__init__(message) - - -class InvalidJson(BaseException): - def __init__(self, message): - # Call the base class constructor with the parameters it needs - super(InvalidJson, self).__init__(message) - - -class JsonImport(Resource): - __urls__ = ( - "/import", - "/import/", - ) - - def _reorder_rules_ids(self, rule, ordered_perimeter_categories_ids, json_data_ids, policy_id, - get_function): - ordered_json_ids = [None] * len(ordered_perimeter_categories_ids) - for json_id in json_data_ids: - data = get_function(self._user_id, policy_id, data_id=json_id) - data = data[0] - if data["category_id"] not in ordered_perimeter_categories_ids: - raise InvalidJson( - "The category id {} of the rule {} does not match the meta rule".format( - data["category_id"], rule)) - if ordered_json_ids[ - ordered_perimeter_categories_ids.index(data["category_id"])] is not None: - raise InvalidJson( - "The category id {} of the rule {} shall not be used twice in the same rule".format( - data["category_id"], rule)) - ordered_json_ids[ordered_perimeter_categories_ids.index(data["category_id"])] = json_id - logger.info(ordered_json_ids) - return ordered_json_ids - - def _import_rules(self, json_rules): - if not isinstance(json_rules, list): - raise InvalidJson("rules shall be a list!") - - for json_rule in json_rules: - json_to_use = dict() - JsonUtils.copy_field_if_exists(json_rule, json_to_use, "instructions", str) - JsonUtils.copy_field_if_exists(json_rule, json_to_use, "enabled", bool, - default_value=True) - - json_ids = dict() - JsonUtils.convert_name_to_id(json_rule, json_ids, "policy", "policy_id", "policy", - PolicyManager, self._user_id) - JsonUtils.convert_name_to_id(json_rule, json_to_use, "meta_rule", "meta_rule_id", - "meta_rule", ModelManager, self._user_id) - json_subject_ids = dict() - json_object_ids = dict() - json_action_ids = dict() - JsonUtils.convert_names_to_ids(json_rule["rule"], json_subject_ids, "subject_data", - "subject", "subject_data", PolicyManager, self._user_id, - json_ids["policy_id"]) - JsonUtils.convert_names_to_ids(json_rule["rule"], json_object_ids, "object_data", - "object", "object_data", PolicyManager, self._user_id, - json_ids["policy_id"]) - JsonUtils.convert_names_to_ids(json_rule["rule"], json_action_ids, "action_data", - "action", "action_data", PolicyManager, self._user_id, - json_ids["policy_id"]) - - meta_rule = ModelManager.get_meta_rules(self._user_id, json_to_use["meta_rule_id"]) - meta_rule = [v for v in meta_rule.values()] - meta_rule = meta_rule[0] - - json_to_use_rule = self._reorder_rules_ids(json_rule, meta_rule["subject_categories"], - json_subject_ids["subject"], - json_ids["policy_id"], - PolicyManager.get_subject_data) - json_to_use_rule = json_to_use_rule + self._reorder_rules_ids(json_rule, meta_rule[ - "object_categories"], json_object_ids["object"], json_ids["policy_id"], - PolicyManager.get_object_data) - json_to_use_rule = json_to_use_rule + self._reorder_rules_ids(json_rule, meta_rule[ - "action_categories"], json_action_ids["action"], json_ids["policy_id"], - PolicyManager.get_action_data) - json_to_use["rule"] = json_to_use_rule - try: - logger.debug("Adding / updating a rule from json {}".format(json_to_use)) - PolicyManager.add_rule(self._user_id, json_ids["policy_id"], - json_to_use["meta_rule_id"], json_to_use) - except exceptions.RuleExisting: - pass - except exceptions.PolicyUnknown: - raise UnknownPolicy("Unknown policy with id {}".format(json_ids["policy_id"])) - - def _import_meta_rules(self, json_meta_rules): - logger.info("Input meta rules : {}".format(json_meta_rules)) - for json_meta_rule in json_meta_rules: - json_to_use = dict() - JsonUtils.copy_field_if_exists(json_meta_rule, json_to_use, "name", str) - JsonUtils.copy_field_if_exists(json_meta_rule, json_to_use, "description", str) - JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "subject_categories", - "subject_categories", "subject_category", ModelManager, - self._user_id) - JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "object_categories", - "object_categories", "object_category", ModelManager, - self._user_id) - JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "action_categories", - "action_categories", "action_category", ModelManager, - self._user_id) - logger.debug("Adding / updating a metarule from json {}".format(json_meta_rule)) - meta_rule = ModelManager.add_meta_rule(self._user_id, meta_rule_id=None, - value=json_to_use) - logger.debug("Added / updated meta rule : {}".format(meta_rule)) - - def _import_subject_object_action_assignments(self, json_item_assignments, type_element): - import_method = getattr(PolicyManager, 'add_' + type_element + '_assignment') - get_method = getattr(PolicyManager, 'get_' + type_element + '_data') - - if not isinstance(json_item_assignments, list): - raise InvalidJson(type_element + " assignments shall be a list!") - - # get the policy id related to the user - policies = PolicyManager.get_policies(self._user_id) - - for json_item_assignment in json_item_assignments: - item_override = JsonUtils.get_override(json_item_assignment) - if item_override is True: - raise ForbiddenOverride( - "{} assignments do not support override flag !".format(type_element)) - - json_assignment = dict() - JsonUtils.convert_name_to_id(json_item_assignment, json_assignment, "category", - "category_id", type_element + "_category", ModelManager, - self._user_id) - - has_found_data = False - # loop over policies - for policy_id in policies: - json_data = dict() - try: - JsonUtils.convert_name_to_id(json_item_assignment, json_assignment, - type_element, "id", type_element, PolicyManager, - self._user_id, policy_id) - JsonUtils.convert_names_to_ids(json_item_assignment, json_data, "assignments", - "data_id", type_element + "_data", PolicyManager, - self._user_id, policy_id, - json_assignment["category_id"]) - has_found_data = True - except UnknownName: - # the category or data has not been found in this policy : we look into the next one - continue - for data_id in json_data["data_id"]: - # find the policy related to the current data - data = get_method(self._user_id, policy_id, data_id, - json_assignment["category_id"]) - if data is not None and len(data) == 1: - logger.debug( - "Adding / updating a {} assignment from json {}".format(type_element, - json_assignment)) - import_method(self._user_id, policy_id, json_assignment["id"], - json_assignment["category_id"], - data_id) - else: - raise UnknownData("Unknown data with id {}".format(data_id)) - - # case the data has not been found in any policies - if has_found_data is False: - raise InvalidJson("The json contains unknown {} data or category : {}".format( - type_element, - json_item_assignment)) - - def _import_subject_object_action_datas(self, json_items_data, mandatory_policy_ids, - type_element): - if type_element == "subject": - import_method = getattr(PolicyManager, 'set_' + type_element + '_data') - else: - import_method = getattr(PolicyManager, 'add_' + type_element + '_data') - # get_method = getattr(PolicyManager, 'get_' + type_element + '_data') - - if not isinstance(json_items_data, list): - raise InvalidJson(type_element + " data shall be a list!") - - for json_item_data in json_items_data: - item_override = JsonUtils.get_override(json_items_data) - if item_override is True: - raise ForbiddenOverride( - "{} datas do not support override flag !".format(type_element)) - json_to_use = dict() - JsonUtils.copy_field_if_exists(json_item_data, json_to_use, "name", str) - JsonUtils.copy_field_if_exists(json_item_data, json_to_use, "description", str) - json_policy = dict() - # field_mandatory : not mandatory if there is some mandatory policies - JsonUtils.convert_names_to_ids(json_item_data, json_policy, "policies", "policy_id", - "policy", - PolicyManager, self._user_id, - field_mandatory=len(mandatory_policy_ids) == 0) - json_category = dict() - JsonUtils.convert_name_to_id(json_item_data, json_category, "category", "category_id", - type_element + "_category", - ModelManager, self._user_id) - policy_ids = [] - if "policy_id" in json_policy: - policy_ids = json_policy["policy_id"] - - for policy_id in policy_ids: - if policy_id is not None and policy_id not in mandatory_policy_ids: - mandatory_policy_ids.append(policy_id) - - if len(mandatory_policy_ids) == 0: - raise InvalidJson("Invalid data, the policy shall be set when importing {}".format( - json_item_data)) - category_id = None - if "category_id" in json_category: - category_id = json_category["category_id"] - if category_id is None: - raise InvalidJson( - "Invalid data, the category shall be set when importing {}".format( - json_item_data)) - - for policy_id in mandatory_policy_ids: - try: - data = import_method(self._user_id, policy_id, category_id=category_id, - value=json_to_use) - except exceptions.PolicyUnknown: - raise UnknownPolicy("Unknown policy with id {}".format(policy_id)) - except Exception as e: - logger.exception(str(e)) - raise e - - def _import_subject_object_action_categories(self, json_item_categories, type_element): - import_method = getattr(ModelManager, 'add_' + type_element + '_category') - get_method = getattr(ModelManager, 'get_' + type_element + '_categories') - - categories = get_method(self._user_id) - - if not isinstance(json_item_categories, list): - raise InvalidJson(type_element + " categories shall be a list!") - - for json_item_category in json_item_categories: - json_to_use = dict() - JsonUtils.copy_field_if_exists(json_item_category, json_to_use, "name", str) - - # check if category with the same name exists : do this in moondb ? - existing_id = None - for category_key in categories: - if categories[category_key]["name"] == json_to_use["name"]: - existing_id = category_key +class JsonImportAPI(object): - JsonUtils.copy_field_if_exists(json_item_category, json_to_use, "description", str) - item_override = JsonUtils.get_override(json_item_category) - if item_override is True: - raise ForbiddenOverride( - "{} categories do not support override flag !".format(type_element)) - - try: - category = import_method(self._user_id, existing_id, json_to_use) - except (exceptions.SubjectCategoryExisting, exceptions.ObjectCategoryExisting, - exceptions.ActionCategoryExisting): - # it already exists: do nothing - logger.warning("Ignored {} category with name {} is already in the database".format( - type_element, json_to_use["name"])) - except Exception as e: - logger.warning("Error while importing the category : {}".format(str(e))) - logger.exception(str(e)) - raise e - - def _import_subject_object_action(self, json_items, mandatory_policy_ids, type_element): - import_method = getattr(PolicyManager, 'add_' + type_element) - get_method = getattr(PolicyManager, 'get_' + type_element + 's') - - if not isinstance(json_items, list): - raise InvalidJson(type_element + " items shall be a list!") - - for json_item in json_items: - json_without_policy_name = dict() - JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "name", str) - JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "description", str) - JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "extra", dict) - JsonUtils.convert_names_to_ids(json_item, json_without_policy_name, "policies", - "policy_list", "policy", PolicyManager, self._user_id, - field_mandatory=False) - policy_ids = json_without_policy_name["policy_list"] - for mandatory_policy_id in mandatory_policy_ids: - if mandatory_policy_id not in policy_ids: - policy_ids.append(mandatory_policy_id) - # policy_ids and json_without_policy_name are references to the same array... - # json_without_policy_name["policy_list"].append(mandatory_policy_id) - - item_override = JsonUtils.get_override(json_item) - if item_override is True: - raise ForbiddenOverride("{} does not support override flag !".format(type_element)) - - if len(policy_ids) == 0: - raise MissingPolicy( - "a {} needs at least one policy to be created or updated : {}".format( - type_element, json.dumps(json_item))) - - for policy_id in policy_ids: - try: - items_in_db = get_method(self._user_id, policy_id) - key = None - for key_in_db in items_in_db: - if items_in_db[key_in_db]["name"] == json_without_policy_name["name"]: - key = key_in_db - break - element = import_method(self._user_id, policy_id, perimeter_id=key, - value=json_without_policy_name) - logger.debug("Added / updated {} : {}".format(type_element, element)) - - except exceptions.PolicyUnknown: - raise UnknownPolicy("Unknown policy when adding a {}!".format(type_element)) - except Exception as e: - logger.exception(str(e)) - raise BaseException(str(e)) - - def _import_policies(self, json_policies): - policy_mandatory_ids = [] - - if not isinstance(json_policies, list): - raise InvalidJson("policies shall be a list!") - - for json_policy in json_policies: - # TODO put this in moondb - # policy_in_db = PolicyManager.get_policies_by_name(json_without_model_name["name"]) - policies = PolicyManager.get_policies(self._user_id) - policy_in_db = None - policy_id = None - for policy_key in policies: - if policies[policy_key]["name"] == json_policy["name"]: - policy_in_db = policies[policy_key] - policy_id = policy_key - # end TODO - if policy_in_db is None: - policy_does_exist = False - else: - policy_does_exist = True - - policy_override = JsonUtils.get_override(json_policy) - policy_mandatory = JsonUtils.get_mandatory(json_policy) - - if policy_override is False and policy_does_exist: - if policy_id: - policy_mandatory_ids.append(policy_id) - logger.warning( - "Existing policy not updated because of the override option is not set !") - continue - - json_without_model_name = dict() - JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "name", str) - JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "description", str) - JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "genre", str) - JsonUtils.convert_name_to_id(json_policy, json_without_model_name, "model", "model_id", - "model", ModelManager, self._user_id, - field_mandatory=False) - - if not policy_does_exist: - logger.debug("Creating policy {} ".format(json_without_model_name)) - added_policy = PolicyManager.add_policy(self._user_id, None, - json_without_model_name) - if policy_mandatory is True: - keys = list(added_policy.keys()) - policy_mandatory_ids.append(keys[0]) - elif policy_override is True: - logger.debug("Updating policy {} ".format(json_without_model_name)) - updated_policy = PolicyManager.update_policy(self._user_id, policy_id, - json_without_model_name) - if policy_mandatory is True: - policy_mandatory_ids.append(policy_id) - return policy_mandatory_ids - - def _import_models_with_new_meta_rules(self, json_models): - if not isinstance(json_models, list): - raise InvalidJson("models shall be a list!") - - for json_model in json_models: - logger.debug("json_model {}".format(json_model)) - models = ModelManager.get_models(self._user_id) - model_in_db = None - model_id = None - for model_key in models: - if ("id" in json_model and model_key == json_model["id"]) or ( - "name" in json_model and models[model_key]["name"] == json_model["name"]): - model_in_db = models[model_key] - model_id = model_key - - # this should not occur as the model has been put in db previously in _import_models_without_new_meta_rules - if model_in_db is None: - raise UnknownModel("Unknown model ") - - json_key = dict() - JsonUtils.convert_names_to_ids(json_model, json_key, "meta_rules", "meta_rule_id", - "meta_rule", ModelManager, self._user_id) - for meta_rule_id in json_key["meta_rule_id"]: - if meta_rule_id not in model_in_db["meta_rules"]: - model_in_db["meta_rules"].append(meta_rule_id) - - ModelManager.update_model(self._user_id, model_id, model_in_db) - - def _import_models_without_new_meta_rules(self, json_models): - if not isinstance(json_models, list): - raise InvalidJson("models shall be a list!") - - for json_model in json_models: - json_without_new_metarules = dict() - JsonUtils.copy_field_if_exists(json_model, json_without_new_metarules, "name", str) - - # TODO put this in moondb - # model_in_db = ModelManager.get_models_by_name(json_without_new_metarules["name"]) - models = ModelManager.get_models(self._user_id) - model_in_db = None - for model_key in models: - if models[model_key]["name"] == json_without_new_metarules["name"]: - model_in_db = models[model_key] - model_id = model_key - # end TODO - - JsonUtils.copy_field_if_exists(json_model, json_without_new_metarules, "description", - str) - if model_in_db is None: - model_does_exist = False - else: - json_without_new_metarules["meta_rules"] = model_in_db["meta_rules"] - model_does_exist = True - model_override = JsonUtils.get_override(json_model) - if not model_does_exist: - logger.debug("Creating model {} ".format(json_without_new_metarules)) - ModelManager.add_model(self._user_id, None, json_without_new_metarules) - elif model_override is True: - logger.debug( - "Updating model with id {} : {} ".format(model_id, json_without_new_metarules)) - ModelManager.update_model(self._user_id, model_id, json_without_new_metarules) - - def _import_pdps(self, json_pdps): - if not isinstance(json_pdps, list): - raise InvalidJson("pdps shall be a list!") - - for json_pdp in json_pdps: - json_to_use = dict() - JsonUtils.copy_field_if_exists(json_pdp, json_to_use, "name", str) - JsonUtils.copy_field_if_exists(json_pdp, json_to_use, "keystone_project_id", str) - JsonUtils.copy_field_if_exists(json_pdp, json_to_use, "security_pipeline", list) - JsonUtils.copy_field_if_exists(json_pdp, json_to_use, "description", str) - - pdps = PDPManager.get_pdp(self._user_id) - exists = False - for pdp_key in pdps: - if pdps[pdp_key]["name"] == json_to_use["name"]: - PDPManager.update_pdp(self._user_id, pdp_id=pdp_key, value=json_to_use) - exists = True - if exists is False: - PDPManager.add_pdp(self._user_id, value=json_to_use) - - def _import_json(self, user_id): - self._user_id = user_id - if 'file' in request.files: - file = request.files['file'] - logger.debug("Importing {} file...".format(file)) - json_content = json.load(file) - else: - json_content = request.json - logger.debug("Importing content: {} ...".format(json_content)) - - # first import the models without the meta rules as they are not yet defined - if "models" in json_content: - logger.info("Importing models...") - self._import_models_without_new_meta_rules(json_content["models"]) - - # import the policies that depends on the models - mandatory_policy_ids = [] - if "policies" in json_content: - logger.info("Importing policies...") - mandatory_policy_ids = self._import_policies(json_content["policies"]) - - # import subjects, subject_data, subject_categories, idem for object and action - list_element = [{"key": "subject"}, {"key": "object"}, {"key": "action"}] - for elt in list_element: - in_key = elt["key"] - key = in_key + "s" - if key in json_content: - logger.info("Importing {}...".format(key)) - self._import_subject_object_action(json_content[key], mandatory_policy_ids, in_key) - key = in_key + "_categories" - if key in json_content: - logger.info("Importing {}...".format(key)) - self._import_subject_object_action_categories(json_content[key], in_key) - - # import meta rules - if "meta_rules" in json_content: - logger.info("Importing meta rules...") - self._import_meta_rules(json_content["meta_rules"]) - - # add the metarule to model - if "models" in json_content: - logger.info("Updating models with meta rules...") - self._import_models_with_new_meta_rules(json_content["models"]) - - for elt in list_element: - in_key = elt["key"] - key = in_key + "_data" - if key in json_content: - logger.info("Importing {}...".format(key)) - self._import_subject_object_action_datas(json_content[key], mandatory_policy_ids, - in_key) - - # import subjects assignments, idem for object and action - for elt in list_element: - in_key = elt["key"] - key = in_key + "_assignments" - if key in json_content: - logger.info("Importing {}...".format(key)) - self._import_subject_object_action_assignments(json_content[key], in_key) - - # import rules - if "rules" in json_content: - logger.info("Importing rules...") - self._import_rules(json_content["rules"]) - - # import pdps - if "pdps" in json_content: - logger.info("Importing pdps...") - self._import_pdps(json_content["pdps"]) - - @check_auth - def post(self, user_id=None): - """Import file. - - :param user_id: user ID who do the request - :return: { - - } - :internal_api: + @staticmethod + @hug.post("/import", requires=api_key_authentication) + def post(request, body, authed_user: hug.directives.user = None): + """ + Import data inside the database + :param request: the request send by the user + :param body: the content of the request + :param authed_user: the name of the authenticated user + :return: "Import ok !" (if import is OK) + :raises multiple exceptions depending on the context """ - self._import_json(user_id) - return "Import ok !" + json_import_ob = JsonImport(driver_name="db", driver=driver) + imported_data = json_import_ob.import_json(moon_user_id=authed_user, request=request, body=body) + LOGGER.info('Imported data: {}'.format(imported_data)) + return imported_data -- cgit 1.2.3-korg