path: root/moon_manager/moon_manager/api/json_import.py

# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors
# This software is distributed under the terms and conditions of the 'Apache-2.0'
# license which can be found in the file 'LICENSE' in this package distribution
# or at 'http://www.apache.org/licenses/LICENSE-2.0'.

from flask import request
from flask_restful import Resource
import flask_restful
from flask import abort

from python_moonutilities.security_functions import check_auth
from python_moonutilities import exceptions
import logging
import json

from moon_manager.api.base_exception import BaseException
from moon_manager.api.json_utils import JsonUtils, UnknownName
from python_moondb.core import PDPManager
from python_moondb.core import PolicyManager
from python_moondb.core import ModelManager


__version__ = "4.5.0"

logger = logging.getLogger("moon.manager.api." + __name__)

INST_CALLBACK = 0
DATA_CALLBACK = 1
ASSIGNMENT_CALLBACK = 2
CATEGORIES_CALLBACK = 3


class ForbiddenOverride(BaseException):
    def __init__(self, message):

        # Call the base class constructor with the parameters it needs
        super(ForbiddenOverride, self).__init__(message)


class UnknownPolicy(BaseException):
    def __init__(self, message):

        # Call the base class constructor with the parameters it needs
        super(UnknownPolicy, self).__init__(message)


class UnknownModel(BaseException):
    def __init__(self, message):

        # Call the base class constructor with the parameters it needs
        super(UnknownModel, self).__init__(message)


class UnknownData(BaseException):
    def __init__(self, message):

        # Call the base class constructor with the parameters it needs
        super(UnknownData, self).__init__(message)


class MissingPolicy(BaseException):
    def __init__(self, message):

        # Call the base class constructor with the parameters it needs
        super(MissingPolicy, self).__init__(message)


class InvalidJson(BaseException):
    def __init__(self, message):

        # Call the base class constructor with the parameters it needs
        super(InvalidJson, self).__init__(message)


class JsonImport(Resource):

    __urls__ = (
        "/import",
        "/import/",
    )

    def _reorder_rules_ids(self, rule, ordered_perimeter_categories_ids, json_data_ids, policy_id, get_function):
        ordered_json_ids = [None]*len(ordered_perimeter_categories_ids)
        logger.info("ordered_json_ids {}".format(ordered_json_ids))
        logger.info("json_data_ids {}".format(json_data_ids))
        for json_id in json_data_ids:
            logger.info("json_id {}".format(json_id))
            data = get_function(self._user_id, policy_id, data_id=json_id)
            data = data[0]
            logger.info("data {}".format(data))
            if data["category_id"] not in ordered_perimeter_categories_ids:
                raise InvalidJson("The category id {} of the rule {} does not match the meta rule".format(data["category_id"], rule))
            if ordered_json_ids[ordered_perimeter_categories_ids.index(data["category_id"])] is not None:
                raise InvalidJson("The category id {} of the rule {} shall not be used twice in the same rule".format(data["category_id"], rule))
            ordered_json_ids[ordered_perimeter_categories_ids.index(data["category_id"])] = json_id
            logger.info(ordered_json_ids)
        return ordered_json_ids

    def _import_rules(self, json_rules):
        if not isinstance(json_rules, list):
            raise InvalidJson("rules shall be a list!")

        for json_rule in json_rules:
            json_to_use = dict()
            JsonUtils.copy_field_if_exists(json_rule, json_to_use, "instructions", str)
            JsonUtils.copy_field_if_exists(json_rule, json_to_use, "enabled", bool, default_value=True)

            json_ids = dict()
            JsonUtils.convert_name_to_id(json_rule, json_ids, "policy", "policy_id", "policy", PolicyManager, self._user_id)
            JsonUtils.convert_name_to_id(json_rule, json_to_use, "meta_rule", "meta_rule_id", "meta_rule", ModelManager, self._user_id)
            json_subject_ids = dict()
            json_object_ids = dict()
            json_action_ids = dict()
            JsonUtils.convert_names_to_ids(json_rule["rule"], json_subject_ids, "subject_data", "subject", "subject_data", PolicyManager, self._user_id, json_ids["policy_id"])
            JsonUtils.convert_names_to_ids(json_rule["rule"], json_object_ids, "object_data", "object", "object_data", PolicyManager, self._user_id, json_ids["policy_id"])
            JsonUtils.convert_names_to_ids(json_rule["rule"], json_action_ids, "action_data", "action", "action_data", PolicyManager, self._user_id, json_ids["policy_id"])

            meta_rule = ModelManager.get_meta_rules(self._user_id, json_to_use["meta_rule_id"])
            meta_rule = [v for v in meta_rule.values()]
            meta_rule = meta_rule[0]

            json_to_use_rule = self._reorder_rules_ids(json_rule, meta_rule["subject_categories"], json_subject_ids["subject"], json_ids["policy_id"], PolicyManager.get_subject_data)
            json_to_use_rule = json_to_use_rule + self._reorder_rules_ids(json_rule, meta_rule["object_categories"], json_object_ids["object"], json_ids["policy_id"], PolicyManager.get_object_data)
            json_to_use_rule = json_to_use_rule + self._reorder_rules_ids(json_rule, meta_rule["action_categories"], json_action_ids["action"], json_ids["policy_id"], PolicyManager.get_action_data)
            json_to_use["rule"] = json_to_use_rule
            try:
                logger.info("Adding / updating a rule from json {}".format(json_to_use))
                PolicyManager.add_rule(self._user_id, json_ids["policy_id"], json_to_use["meta_rule_id"], json_to_use)
            except exceptions.RuleExisting:
                pass
            except exceptions.PolicyUnknown:
                raise UnknownPolicy("Unknown policy with id {}".format(json_ids["policy_id"]))

    def _import_meta_rules(self, json_meta_rules):
        logger.info("Input meta rules : {}".format(json_meta_rules))
        for json_meta_rule in json_meta_rules:
            json_to_use = dict()
            logger.info("Input meta rule : {}".format(json_meta_rule))
            JsonUtils.copy_field_if_exists(json_meta_rule, json_to_use, "name", str)
            JsonUtils.copy_field_if_exists(json_meta_rule, json_to_use, "description", str)
            JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "subject_categories", "subject_categories", "subject_category", ModelManager, self._user_id)
            JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "object_categories", "object_categories", "object_category", ModelManager, self._user_id)
            JsonUtils.convert_names_to_ids(json_meta_rule, json_to_use, "action_categories", "action_categories", "action_category", ModelManager, self._user_id)
            logger.info("Adding / updating a metarule from json {}".format(json_meta_rule))
            meta_rule = ModelManager.add_meta_rule(self._user_id, meta_rule_id=None, value=json_to_use)
            logger.info("Added / updated meta rule : {}".format(meta_rule))

    def _import_subject_object_action_assignments(self, json_item_assignments, type_element):
        import_method = getattr(PolicyManager, 'add_' + type_element + '_assignment')
        get_method = getattr(PolicyManager, 'get_' + type_element + '_data')

        if not isinstance(json_item_assignments, list):
            raise InvalidJson(type_element + " assignments shall be a list!")

        # get the policy id related to the user
        policies = PolicyManager.get_policies(self._user_id)

        for json_item_assignment in json_item_assignments:
            item_override = JsonUtils.get_override(json_item_assignment)
            if item_override is True:
                raise ForbiddenOverride("{} assignments do not support override flag !".format(type_element))

            json_assignment = dict()
            JsonUtils.convert_name_to_id(json_item_assignment, json_assignment, "category", "category_id", type_element + "_category", ModelManager, self._user_id)

            has_found_data = False
            # loop over policies
            for policy_id in policies:
                json_data = dict()
                try:
                    JsonUtils.convert_name_to_id(json_item_assignment, json_assignment, type_element, "id", type_element, PolicyManager, self._user_id, policy_id)
                    JsonUtils.convert_names_to_ids(json_item_assignment, json_data, "assignments", "data_id", type_element + "_data", PolicyManager, self._user_id, policy_id, json_assignment["category_id"])
                    has_found_data = True
                except UnknownName:
                    # the category or data has not been found in this policy : we look into the next one
                    continue
                for data_id in json_data["data_id"]:
                    # find the policy related to the current data
                    data = get_method(self._user_id, policy_id, data_id, json_assignment["category_id"])
                    if data is not None and len(data) == 1:
                        logger.info("Adding / updating a {} assignment from json {}".format(type_element, json_assignment))
                        import_method(self._user_id, policy_id, json_assignment["id"], json_assignment["category_id"], data_id)
                    else:
                        raise UnknownData("Unknown data with id {}".format(data_id))

            # case the data has not been found in any policies
            if has_found_data is False:
                raise InvalidJson("The json contains unknown {} data or category : {}".format(type_element,json_item_assignment))

    def _import_subject_object_action_datas(self, json_items_data,  mandatory_policy_ids, type_element):
        if type_element == "subject":
            import_method = getattr(PolicyManager, 'set_' + type_element + '_data')
        else:
            import_method = getattr(PolicyManager, 'add_' + type_element + '_data')
        # get_method = getattr(PolicyManager, 'get_' + type_element + '_data')

        if not isinstance(json_items_data, list):
            raise InvalidJson(type_element + " data shall be a list!")

        for json_item_data in json_items_data:
            item_override = JsonUtils.get_override(json_items_data)
            if item_override is True:
                raise ForbiddenOverride("{} datas do not support override flag !".format(type_element))
            logger.info("json_item_data {}".format(json_item_data))
            json_to_use = dict()
            JsonUtils.copy_field_if_exists(json_item_data, json_to_use, "name", str)
            JsonUtils.copy_field_if_exists(json_item_data, json_to_use, "description", str)
            json_policy = dict()
            # field_mandatory : not mandatory if there is some mandatory policies
            JsonUtils.convert_names_to_ids(json_item_data, json_policy, "policies", "policy_id", "policy",
                                         PolicyManager, self._user_id, field_mandatory=len(mandatory_policy_ids) == 0)
            logger.info("json_policy {}".format(json_policy))
            json_category = dict()
            JsonUtils.convert_name_to_id(json_item_data, json_category, "category", "category_id", type_element+"_category",
                                         ModelManager, self._user_id)
            logger.info("json_category {}".format(json_category))
            policy_ids = []
            if "policy_id" in json_policy:
                policy_ids = json_policy["policy_id"]

            for policy_id in policy_ids:
                if policy_id is not None and policy_id not in mandatory_policy_ids:
                    mandatory_policy_ids.append(policy_id)

            if len(mandatory_policy_ids) == 0:
                raise InvalidJson("Invalid data, the policy shall be set when importing {}".format(json_item_data))
            category_id = None
            if "category_id" in json_category:
                category_id = json_category["category_id"]
            if category_id is None:
                raise InvalidJson("Invalid data, the category shall be set when importing {}".format(json_item_data))

            for policy_id in mandatory_policy_ids:
                try:
                    # existing_datas = get_method(self._user_id, policy_id,category_id=category_id)
                    # logger.info(existing_datas)
                    logger.info("Adding / updating a {} data with policy id {} and category id {} from json {}".format(type_element, policy_id, category_id, json_to_use))
                    data = import_method(self._user_id, policy_id,  category_id=category_id, value=json_to_use)
                    logger.info("Added / updated {} data : {}".format(type_element, data))
                except exceptions.PolicyUnknown:
                    raise UnknownPolicy("Unknown policy with id {}".format(policy_id))
                except Exception as e:
                    raise BaseException(str(e))

    def _import_subject_object_action_categories(self, json_item_categories, type_element):
        import_method = getattr(ModelManager, 'add_' + type_element + '_category')
        get_method = getattr(ModelManager, 'get_' + type_element + '_categories')

        categories = get_method(self._user_id)

        if not isinstance(json_item_categories, list):
            raise InvalidJson(type_element + " categories shall be a list!")

        for json_item_category in json_item_categories:
            json_to_use = dict()
            JsonUtils.copy_field_if_exists(json_item_category, json_to_use, "name", str)

            # check if category with the same name exists : do this in moondb ?
            existing_id = None
            for category_key in categories:
                if categories[category_key]["name"] == json_to_use["name"]:
                    existing_id = category_key

            JsonUtils.copy_field_if_exists(json_item_category, json_to_use, "description", str)
            item_override = JsonUtils.get_override(json_item_category)
            if item_override is True:
                raise ForbiddenOverride("{} categories do not support override flag !".format(type_element))

            try:
                logger.info("Adding a {} category from json {}".format(type_element, json_to_use))
                category = import_method(self._user_id, existing_id, json_to_use)
                logger.info("Added category {}".format(category))
            except (exceptions.SubjectCategoryExisting, exceptions.ObjectCategoryExisting, exceptions.ActionCategoryExisting):
                # it already exists: do nothing
                logger.info("Ignored {} category with name {} is already in the database".format(type_element, json_to_use["name"]))
            except Exception as e:
                logger.info("Error while importing the category : {}".format(str(e)))
                raise e

    def _import_subject_object_action(self, json_items, mandatory_policy_ids, type_element):
        import_method = getattr(PolicyManager, 'add_' + type_element)
        get_method = getattr(PolicyManager, 'get_' + type_element + 's')

        if not isinstance(json_items, list):
            raise InvalidJson(type_element + " items shall be a list!")

        for json_item in json_items:
            json_without_policy_name = dict()
            JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "name", str)
            JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "description", str)
            JsonUtils.copy_field_if_exists(json_item, json_without_policy_name, "extra", dict)
            JsonUtils.convert_names_to_ids(json_item, json_without_policy_name, "policies", "policy_list", "policy", PolicyManager, self._user_id, field_mandatory=False)
            policy_ids = json_without_policy_name["policy_list"]
            for mandatory_policy_id in mandatory_policy_ids:
                if mandatory_policy_id not in policy_ids:
                    policy_ids.append(mandatory_policy_id)
                    # policy_ids and json_without_policy_name are references to the same array...
                    # json_without_policy_name["policy_list"].append(mandatory_policy_id)

            item_override = JsonUtils.get_override(json_item)
            if item_override is True:
                raise ForbiddenOverride("{} does not support override flag !".format(type_element))

            if len(policy_ids) == 0:
                raise MissingPolicy("a {} needs at least one policy to be created or updated : {}".format(type_element,  json.dumps(json_item)))

            for policy_id in policy_ids:
                try:
                    items_in_db = get_method(self._user_id, policy_id)
                    key = None
                    for key_in_db in items_in_db:
                        if items_in_db[key_in_db]["name"] == json_without_policy_name["name"]:
                            key = key_in_db
                            break
                    if key is None:
                        logger.info("Adding a {} from json {} to the policy with id {}".format(type_element, json_without_policy_name, policy_id))
                    else:
                        logger.info("Updating a {} from json {} to the policy with id {}".format(type_element, json_without_policy_name, policy_id))
                    element = import_method(self._user_id, policy_id, perimeter_id=key, value=json_without_policy_name)
                    logger.info("Added / updated {} : {}".format(type_element, element))

                except exceptions.PolicyUnknown:
                    raise UnknownPolicy("Unknown policy when adding a {}!".format(type_element))
                except Exception as e:
                    raise BaseException(str(e))

    def _import_policies(self, json_policies):
        policy_mandatory_ids = []

        if not isinstance(json_policies, list):
            raise InvalidJson("policies shall be a list!")

        for json_policy in json_policies:
            # TODO put this in moondb
            # policy_in_db = PolicyManager.get_policies_by_name(json_without_model_name["name"])
            policies = PolicyManager.get_policies(self._user_id)
            policy_in_db = None
            logger.info(policies)
            for policy_key in policies:
                if policies[policy_key]["name"] == json_policy["name"]:
                    policy_in_db = policies[policy_key]
                    policy_id = policy_key
            # end TODO
            if policy_in_db is None:
                policy_does_exist = False
            else:
                policy_does_exist = True

            policy_override = JsonUtils.get_override(json_policy)
            policy_mandatory = JsonUtils.get_mandatory(json_policy)

            if policy_override is False and policy_does_exist:
                policy_mandatory_ids.append(policy_id)
                logger.warning("Existing policy not updated because of the override option is not set !")
                continue

            json_without_model_name = dict()
            JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "name", str)
            JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "description", str)
            JsonUtils.copy_field_if_exists(json_policy, json_without_model_name, "genre", str)
            JsonUtils.convert_name_to_id(json_policy, json_without_model_name, "model", "model_id", "model", ModelManager, self._user_id, field_mandatory=False)

            if not policy_does_exist:
                logger.info("Creating policy {} ".format(json_without_model_name))
                added_policy = PolicyManager.add_policy(self._user_id, None, json_without_model_name)
                logger.info("Added policy {}".format(added_policy))
                if policy_mandatory is True:
                    keys = list(added_policy.keys())
                    policy_mandatory_ids.append(keys[0])
            elif policy_override is True:
                logger.info("Updating policy {} ".format(json_without_model_name))
                updated_policy = PolicyManager.update_policy(self._user_id, policy_id, json_without_model_name)
                logger.info("Updated policy {}".format(updated_policy))
                if policy_mandatory is True:
                    policy_mandatory_ids.append(policy_id)
        return policy_mandatory_ids

    def _import_models_with_new_meta_rules(self, json_models):
        if not isinstance(json_models, list):
            raise InvalidJson("models shall be a list!")

        for json_model in json_models:
            logger.info("json_model {}".format(json_model))
            models = ModelManager.get_models(self._user_id)
            model_in_db = None
            model_id = None
            for model_key in models:
                if ("id" in json_model and model_key == json_model["id"]) or ("name" in json_model and models[model_key]["name"] == json_model["name"]):
                    model_in_db = models[model_key]
                    model_id = model_key

            logger.info("model in db".format(model_in_db))
            # this should not occur as the model has been put in db previously in _import_models_without_new_meta_rules
            if model_in_db is None:
                raise UnknownModel("Unknwon model ")

            json_key = dict()
            JsonUtils.convert_names_to_ids(json_model, json_key, "meta_rules", "meta_rule_id", "meta_rule", ModelManager, self._user_id)
            logger.info("json_key {}".format(json_key))
            for meta_rule_id in json_key["meta_rule_id"]:
                if meta_rule_id not in model_in_db["meta_rules"]:
                    model_in_db["meta_rules"].append(meta_rule_id)

            logger.info("Updating model with id {} : {} ".format(model_id, model_in_db))
            ModelManager.update_model(self._user_id, model_id, model_in_db)

    def _import_models_without_new_meta_rules(self, json_models):
        if not isinstance(json_models, list):
            raise InvalidJson("models shall be a list!")

        for json_model in json_models:
            json_without_new_metarules = dict()
            JsonUtils.copy_field_if_exists(json_model, json_without_new_metarules, "name", str)

            # TODO put this in moondb
            # model_in_db = ModelManager.get_models_by_name(json_without_new_metarules["name"])
            models = ModelManager.get_models(self._user_id)
            model_in_db = None
            for model_key in models:
                if models[model_key]["name"] == json_without_new_metarules["name"]:
                    model_in_db = models[model_key]
                    model_id = model_key
            # end TODO

            JsonUtils.copy_field_if_exists(json_model, json_without_new_metarules, "description", str)
            if model_in_db is None:
                model_does_exist = False
            else:
                logger.info("model_in_db {}".format(model_in_db))
                # JsonUtils.convert_names_to_ids(model_in_db, json_without_new_metarules, "meta_rules", "meta_rule_id", "meta_rule", ModelManager, self._user_id)
                json_without_new_metarules["meta_rule_id"] = model_in_db["meta_rules"]
                model_does_exist = True
            model_override = JsonUtils.get_override(json_model)
            if not model_does_exist:
                logger.info("Creating model {} ".format(json_without_new_metarules))
                ModelManager.add_model(self._user_id, None, json_without_new_metarules)
            elif model_override is True:
                logger.info("Updating model with id {} : {} ".format(model_id, json_without_new_metarules))
                ModelManager.update_model(self._user_id, model_id, json_without_new_metarules)

    def _import_pdps(self, json_pdps):
        if not isinstance(json_pdps, list):
            raise InvalidJson("pdps shall be a list!")

        for json_pdp in json_pdps:
            json_to_use = dict()
            JsonUtils.copy_field_if_exists(json_pdp, json_to_use, "name", str)
            JsonUtils.copy_field_if_exists(json_pdp, json_to_use, "keystone_project_id", str)
            JsonUtils.copy_field_if_exists(json_pdp, json_to_use, "security_pipeline", list)
            JsonUtils.copy_field_if_exists(json_pdp, json_to_use, "description", str)

            pdps = PDPManager.get_pdp(self._user_id)
            exists = False
            for pdp_key in pdps:
                if pdps[pdp_key]["name"] == json_to_use["name"]:
                    PDPManager.update_pdp(self._user_id, pdp_id=pdp_key, value=json_to_use)
                    exists = True
            if exists is False:
                PDPManager.add_pdp(self._user_id, value=json_to_use)

    def _import_json(self, user_id):
        self._user_id = user_id
        if 'file' in request.files:
            file = request.files['file']
            logger.info("Importing {} file...".format(file))
            json_content = json.load(file)
        else:
            json_content = request.json
        logger.info("Importing content:  {} ...".format(json_content))

        # first import the models without the meta rules as they are not yet defined
        if "models" in json_content:
            logger.info("Importing models...")
            self._import_models_without_new_meta_rules(json_content["models"])

        # import the policies that depends on the models
        mandatory_policy_ids = []
        if "policies" in json_content:
            logger.info("Importing policies...")
            mandatory_policy_ids = self._import_policies(json_content["policies"])

        # import subjects, subject_data, subject_categories, idem for object and action
        list_element = [{"key": "subject"}, {"key": "object"}, {"key": "action"}]
        for elt in list_element:
            in_key = elt["key"]
            key = in_key + "s"
            if key in json_content:
                logger.info("Importing {}...".format(key))
                self._import_subject_object_action(json_content[key], mandatory_policy_ids, in_key)
            key = in_key + "_categories"
            if key in json_content:
                logger.info("Importing {}...".format(key))
                self._import_subject_object_action_categories(json_content[key], in_key)
            key = in_key + "_data"
            if key in json_content:
                logger.info("Importing {}...".format(key))
                self._import_subject_object_action_datas(json_content[key], mandatory_policy_ids, in_key)

        # import meta rules
        if "meta_rules" in json_content:
            logger.info("Importing meta rules...")
            self._import_meta_rules(json_content["meta_rules"])

        # add the metarule to model
        if "models" in json_content:
            logger.info("Updating models with meta rules...")
            self._import_models_with_new_meta_rules(json_content["models"])

        # import subjects assignments, idem for object and action
        for elt in list_element:
            in_key = elt["key"]
            key = in_key + "_assignments"
            if key in json_content:
                logger.info("Importing {}...".format(key))
                self._import_subject_object_action_assignments(json_content[key], in_key)

        # import rules
        if "rules" in json_content:
            logger.info("Importing rules...")
            self._import_rules(json_content["rules"])

        # import pdps
        if "pdps" in json_content:
            logger.info("Importing pdps...")
            self._import_pdps(json_content["pdps"])

    @check_auth
    def post(self, user_id=None):
        """Import file.

        :param user_id: user ID who do the request
        :return: {

        }
        :internal_api:
        """
        self._import_json(user_id)
        return "Import ok !"