moon v4 re-organization

Change-Id: I73665f739f35ae18175f98d0739567e403c1fa80
Signed-off-by: RHE <rebirthmonkey@gmail.com>

13 files changed, 1725 insertions, 0 deletions

diff --git a/moon_forming/Dockerfile b/moon_forming/Dockerfile
new file mode 100644
index 00000000..fe48eee0
--- /dev/null
+++ b/moon_forming/Dockerfile
@@ -0,0 +1,10 @@
+FROM python:3
+WORKDIR /usr/src/app
+RUN pip install --no-cache-dir --upgrade requests pyyaml python_moonutilities python_moondb
+
+ENV POPULATE_ARGS "-v"
+
+ADD . /root
+WORKDIR /root
+
+CMD sh /root/run.sh ${POPULATE_ARGS}
\ No newline at end of file
diff --git a/moon_forming/README.md b/moon_forming/README.md
new file mode 100644
index 00000000..d71805cb
--- /dev/null
+++ b/moon_forming/README.md
@@ -0,0 +1,44 @@
+# Moon Forming 
+moon_forming is a container to automatize the configuration of the Moon platform
+
+## Run
+```bash
+docker run wukongsun/moon_forming:latest
+```
+
+## Consul
+The Moon platform is already configured after the installation.
+If you want to see or modify the configuration, go with a web browser 
+to the following page: `http://localhost:30006`.
+
+With the consul server, you can update the configuration in the `KEY/VALUE` tab.
+There are some configuration items, lots of them are only read when a new K8S pod is started
+and not during its life cycle.
+
+**WARNING: some confidential information are put here in clear text.
+This is a known security issue.**
+
+### Keystone
+If you have your own Keystone server, you can point Moon to your Keystone in the 
+`openstack/keystone` element: `http://localhost:30005/ui/#/dc1/kv/openstack/keystone/edit`.
+This configuration element is read every time Moon need it, specially when adding users.
+
+### Database
+The database can also be modified through: `http://localhost:30005/ui/#/dc1/kv/database/edit`.
+
+**WARNING: the password is in clear text, this is a known security issue.**
+
+If you want to use your own database server, change the configuration:
+
+    {"url": "mysql+pymysql://my_user:my_secret_password@my_server/moon", "driver": "sql"}
+
+Then you have to rebuild the database before using it. 
+This can be done with the following commands:
+```bash
+kubectl delete -f $MOON/tools/moon_kubernetes/templates/moon_configuration.yaml
+kubectl create -f $MOON/tools/moon_kubernetes/templates/moon_configuration.yaml
+```
+
+
+
+
diff --git a/moon_forming/conf/mls.py b/moon_forming/conf/mls.py
new file mode 100644
index 00000000..0e6285c9
--- /dev/null
+++ b/moon_forming/conf/mls.py
@@ -0,0 +1,59 @@
+
+pdp_name = "pdp_mls"
+policy_name = "MLS Policy example"
+model_name = "MLS"
+policy_genre = "authz"
+
+subjects = {"adminuser": "", "user1": "", "user2": "", }
+objects = {"vm0": "", "vm1": "", }
+actions = {"start": "", "stop": ""}
+
+subject_categories = {"subject-security-level": "", }
+object_categories = {"object-security-level": "", }
+action_categories = {"action-type": "", }
+
+subject_data = {
+    "subject-security-level": {"low": "", "medium": "", "high": ""},
+}
+object_data = {
+    "object-security-level": {"low": "", "medium": "", "high": ""},
+}
+action_data = {"action-type": {"vm-action": "", "storage-action": "", }}
+
+subject_assignments = {
+    "adminuser": {"subject-security-level": "high"},
+    "user1": {"subject-security-level": "medium"},
+}
+object_assignments = {
+    "vm0": {"object-security-level": "medium"},
+    "vm1": {"object-security-level": "low"},
+}
+action_assignments = {
+    "start": {"action-type": "vm-action"},
+    "stop": {"action-type": "vm-action"}
+}
+
+meta_rule = {
+    "mls": {
+        "id": "",
+        "value": ("subject-security-level",
+                  "object-security-level",
+                  "action-type")},
+}
+
+rules = {
+    "mls": (
+        {
+            "rule": ("high", "medium", "vm-action"),
+            "instructions": ({"decision": "grant"})
+        },
+        {
+            "rule": ("high", "low", "vm-action"),
+            "instructions": ({"decision": "grant"})
+        },
+        {
+            "rule": ("medium", "low", "vm-action"),
+            "instructions": ({"decision": "grant"})
+        },
+    )
+}
diff --git a/moon_forming/conf/rbac.py b/moon_forming/conf/rbac.py
new file mode 100644
index 00000000..25c010fd
--- /dev/null
+++ b/moon_forming/conf/rbac.py
@@ -0,0 +1,61 @@
+
+pdp_name = "pdp_rbac"
+policy_name = "RBAC policy example"
+model_name = "RBAC"
+policy_genre = "authz"
+
+subjects = {"adminuser": "", "user1": "", }
+objects = {"vm0": "", "vm1": "", }
+actions = {"start": "", "stop": ""}
+
+subject_categories = {"role": "", }
+object_categories = {"id": "", }
+action_categories = {"action-type": "", }
+
+subject_data = {"role": {"admin": "", "employee": "", "*": ""}}
+object_data = {"id": {"vm0": "", "vm1": "", "*": ""}}
+action_data = {"action-type": {"vm-action": "", "*": ""}}
+
+subject_assignments = {
+    "adminuser":
+        ({"role": "admin"}, {"role": "employee"}, {"role": "*"}),
+    "user1":
+        ({"role": "employee"}, {"role": "*"}),
+}
+object_assignments = {
+    "vm0":
+        ({"id": "vm0"}, {"id": "*"}),
+    "vm1":
+        ({"id": "vm1"}, {"id": "*"})
+}
+action_assignments = {
+    "start":
+        ({"action-type": "vm-action"}, {"action-type": "*"}),
+    "stop":
+        ({"action-type": "vm-action"}, {"action-type": "*"})
+}
+
+meta_rule = {
+    "rbac": {"id": "", "value": ("role", "id", "action-type")},
+}
+
+rules = {
+    "rbac": (
+        {
+            "rule": ("admin", "vm0", "vm-action"),
+            "instructions": (
+                {"decision": "grant"},
+                # "grant" to immediately exit,
+                # "continue" to wait for the result of next policy
+            )
+        },
+        {
+            "rule": ("employee", "vm1", "vm-action"),
+            "instructions": (
+                {"decision": "grant"},
+            )
+        },
+    )
+}
+
+
diff --git a/moon_forming/conf2consul.py b/moon_forming/conf2consul.py
new file mode 100644
index 00000000..46c99d5c
--- /dev/null
+++ b/moon_forming/conf2consul.py
@@ -0,0 +1,103 @@
+import os
+import sys
+import requests
+import yaml
+import logging
+import json
+import base64
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger("moon.conf2consul")
+requests_log = logging.getLogger("requests.packages.urllib3")
+requests_log.setLevel(logging.WARNING)
+requests_log.propagate = True
+
+if len(sys.argv) == 2:
+    if os.path.isfile(sys.argv[1]):
+        CONF_FILENAME = sys.argv[1]
+        CONSUL_HOST = "consul"
+    else:
+        CONF_FILENAME = "moon.conf"
+        CONSUL_HOST = sys.argv[1]
+    CONSUL_PORT = 8500
+else:
+    CONSUL_HOST = sys.argv[1] if len(sys.argv) > 1 else "consul"
+    CONSUL_PORT = sys.argv[2] if len(sys.argv) > 2 else 8500
+    CONF_FILENAME = sys.argv[3] if len(sys.argv) > 3 else "moon.conf"
+HEADERS = {"content-type": "application/json"}
+
+
+def search_config_file():
+    data_config = None
+    for _file in (
+            CONF_FILENAME,
+            "conf/moon.conf",
+            "../moon.conf",
+            "../conf/moon.conf",
+            "/etc/moon/moon.conf",
+    ):
+        try:
+            data_config = yaml.safe_load(open(_file))
+        except FileNotFoundError:
+            data_config = None
+            continue
+        else:
+            break
+    if not data_config:
+        raise Exception("Configuration file not found...")
+    return data_config
+
+
+def put(key, value):
+    url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key)
+    log.info(url)
+    req = requests.put(
+        url,
+        headers=HEADERS,
+        json=value
+    )
+    if req.status_code != 200:
+        raise Exception("Error connecting to Consul ({}, {})".format(req.status_code, req.text))
+
+
+def get(key):
+    url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key)
+    req = requests.get(url)
+    data = req.json()
+    for item in data:
+        log.info("{} {} -> {}".format(
+            req.status_code,
+            item["Key"],
+            json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+        ))
+        yield json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+
+
+def main():
+    data_config = search_config_file()
+    req = requests.head("http://{}:{}/ui/".format(CONSUL_HOST, CONSUL_PORT))
+    if req.status_code != 200:
+        log.critical("Consul is down...")
+        log.critical("request info: {}/{}".format(req, req.text))
+        sys.exit(1)
+
+    put("database", data_config["database"])
+    # put("messenger", data_config["messenger"])
+    # put("slave", data_config["slave"])
+    # put("docker", data_config["docker"])
+    put("logging", data_config["logging"])
+    put("components_port_start", data_config["components"]["port_start"])
+
+    for _key, _value in data_config["components"].items():
+        if type(_value) is dict:
+            put("components/{}".format(_key), data_config["components"][_key])
+
+    for _key, _value in data_config["plugins"].items():
+        put("plugins/{}".format(_key), data_config["plugins"][_key])
+
+    for _key, _value in data_config["openstack"].items():
+        put("openstack/{}".format(_key), data_config["openstack"][_key])
+
+
+main()
+
diff --git a/moon_forming/moon.conf b/moon_forming/moon.conf
new file mode 100644
index 00000000..dc498e34
--- /dev/null
+++ b/moon_forming/moon.conf
@@ -0,0 +1,79 @@
+database:
+    url: mysql+pymysql://moon:p4sswOrd1@db/moon
+    driver: sql
+
+openstack:
+    keystone:
+        url: http://keystone:5000/v3
+        user: admin
+        password: p4ssw0rd
+        domain: default
+        project: admin
+        check_token: false
+        certificate: false
+
+plugins:
+    authz:
+        container: wukongsun/moon_authz:v4.3
+        port: 8081
+    session:
+        container: asteroide/session:latest
+        port: 8082
+
+components:
+    interface:
+        port: 8080
+        bind: 0.0.0.0
+        hostname: interface
+        container: wukongsun/moon_interface:v4.3
+    orchestrator:
+        port: 8083
+        bind: 0.0.0.0
+        hostname: orchestrator
+        container: wukongsun/moon_orchestrator:v4.3
+    wrapper:
+        port: 8080
+        bind: 0.0.0.0
+        hostname: wrapper
+        container: wukongsun/moon_wrapper:v4.3.1
+        timeout: 5
+    manager:
+        port: 8082
+        bind: 0.0.0.0
+        hostname: manager
+        container: wukongsun/moon_manager:v4.3.1
+    port_start: 31001
+
+logging:
+  version: 1
+
+  formatters:
+    brief:
+      format: "%(levelname)s %(name)s %(message)-30s"
+    custom:
+      format: "%(asctime)-15s %(levelname)s %(name)s %(message)s"
+
+  handlers:
+    console:
+      class : logging.StreamHandler
+      formatter: brief
+      level   : INFO
+      stream  : ext://sys.stdout
+    file:
+      class : logging.handlers.RotatingFileHandler
+      formatter: custom
+      level   : DEBUG
+      filename: /tmp/moon.log
+      maxBytes: 1048576
+      backupCount: 3
+
+  loggers:
+    moon:
+      level: DEBUG
+      handlers: [console, file]
+      propagate: no
+
+  root:
+    level: ERROR
+    handlers: [console]
+
diff --git a/moon_forming/populate_default_values.py b/moon_forming/populate_default_values.py
new file mode 100644
index 00000000..fa099458
--- /dev/null
+++ b/moon_forming/populate_default_values.py
@@ -0,0 +1,235 @@
+import argparse
+import logging
+from importlib.machinery import SourceFileLoader
+from utils.pdp import *
+from utils.models import *
+from utils.policies import *
+
+parser = argparse.ArgumentParser()
+parser.add_argument('filename', help='scenario filename', nargs=1)
+parser.add_argument("--verbose", "-v", action='store_true',
+                    help="verbose mode")
+parser.add_argument("--debug", "-d", action='store_true', help="debug mode")
+parser.add_argument("--keystone-pid", "-k", dest="keystone_pid", default="",
+                    help="Force a particular Keystone Project ID")
+args = parser.parse_args()
+
+FORMAT = '%(asctime)-15s %(levelname)s %(message)s'
+if args.debug:
+    logging.basicConfig(
+        format=FORMAT,
+        level=logging.DEBUG)
+elif args.verbose:
+    logging.basicConfig(
+        format=FORMAT,
+        level=logging.INFO)
+else:
+    logging.basicConfig(
+        format=FORMAT,
+        level=logging.WARNING)
+
+requests_log = logging.getLogger("requests.packages.urllib3")
+requests_log.setLevel(logging.WARNING)
+requests_log.propagate = True
+
+logger = logging.getLogger("moonforming")
+
+if args.filename:
+    print("Loading: {}".format(args.filename[0]))
+
+m = SourceFileLoader("scenario", args.filename[0])
+
+scenario = m.load_module()
+
+
+def create_model(model_id=None):
+    if args.verbose:
+        logger.info("Creating model {}".format(scenario.model_name))
+    if not model_id:
+        logger.info("Add model")
+        model_id = add_model(name=scenario.model_name)
+    logger.info("Add subject categories")
+    for cat in scenario.subject_categories:
+        scenario.subject_categories[cat] = add_subject_category(name=cat)
+    logger.info("Add object categories")
+    for cat in scenario.object_categories:
+        scenario.object_categories[cat] = add_object_category(name=cat)
+    logger.info("Add action categories")
+    for cat in scenario.action_categories:
+        scenario.action_categories[cat] = add_action_category(name=cat)
+    sub_cat = []
+    ob_cat = []
+    act_cat = []
+    meta_rule_list = []
+    for item_name, item_value in scenario.meta_rule.items():
+        for item in item_value["value"]:
+            if item in scenario.subject_categories:
+                sub_cat.append(scenario.subject_categories[item])
+            elif item in scenario.object_categories:
+                ob_cat.append(scenario.object_categories[item])
+            elif item in scenario.action_categories:
+                act_cat.append(scenario.action_categories[item])
+        meta_rules = check_meta_rule(meta_rule_id=None)
+        for _meta_rule_id, _meta_rule_value in meta_rules['meta_rules'].items():
+            if _meta_rule_value['name'] == item_name:
+                meta_rule_id = _meta_rule_id
+                break
+        else:
+            logger.info("Add meta rule")
+            meta_rule_id = add_meta_rule(item_name, sub_cat, ob_cat, act_cat)
+        item_value["id"] = meta_rule_id
+        if meta_rule_id not in meta_rule_list:
+            meta_rule_list.append(meta_rule_id)
+    return model_id, meta_rule_list
+
+
+def create_policy(model_id, meta_rule_list):
+    if args.verbose:
+        logger.info("Creating policy {}".format(scenario.policy_name))
+    _policies = check_policy()
+    for _policy_id, _policy_value in _policies["policies"].items():
+        if _policy_value['name'] == scenario.policy_name:
+            policy_id = _policy_id
+            break
+    else:
+        policy_id = add_policy(name=scenario.policy_name, genre=scenario.policy_genre)
+
+    update_policy(policy_id, model_id)
+
+    for meta_rule_id in meta_rule_list:
+        logger.debug("add_meta_rule_to_model {} {}".format(model_id, meta_rule_id))
+        add_meta_rule_to_model(model_id, meta_rule_id)
+
+    logger.info("Add subject data")
+    for subject_cat_name in scenario.subject_data:
+        for subject_data_name in scenario.subject_data[subject_cat_name]:
+            data_id = scenario.subject_data[subject_cat_name][subject_data_name] = add_subject_data(
+                policy_id=policy_id,
+                category_id=scenario.subject_categories[subject_cat_name], name=subject_data_name)
+            scenario.subject_data[subject_cat_name][subject_data_name] = data_id
+    logger.info("Add object data")
+    for object_cat_name in scenario.object_data:
+        for object_data_name in scenario.object_data[object_cat_name]:
+            data_id = scenario.object_data[object_cat_name][object_data_name] = add_object_data(
+                policy_id=policy_id,
+                category_id=scenario.object_categories[object_cat_name], name=object_data_name)
+            scenario.object_data[object_cat_name][object_data_name] = data_id
+    logger.info("Add action data")
+    for action_cat_name in scenario.action_data:
+        for action_data_name in scenario.action_data[action_cat_name]:
+            data_id = scenario.action_data[action_cat_name][action_data_name] = add_action_data(
+                policy_id=policy_id,
+                category_id=scenario.action_categories[action_cat_name], name=action_data_name)
+            scenario.action_data[action_cat_name][action_data_name] = data_id
+
+    logger.info("Add subjects")
+    for name in scenario.subjects:
+        scenario.subjects[name] = add_subject(policy_id, name=name)
+    logger.info("Add objects")
+    for name in scenario.objects:
+        scenario.objects[name] = add_object(policy_id, name=name)
+    logger.info("Add actions")
+    for name in scenario.actions:
+        scenario.actions[name] = add_action(policy_id, name=name)
+
+    logger.info("Add subject assignments")
+    for subject_name in scenario.subject_assignments:
+        if type(scenario.subject_assignments[subject_name]) in (list, tuple):
+            for items in scenario.subject_assignments[subject_name]:
+                for subject_category_name in items:
+                    subject_id = scenario.subjects[subject_name]
+                    subject_cat_id = scenario.subject_categories[subject_category_name]
+                    for data in scenario.subject_assignments[subject_name]:
+                        subject_data_id = scenario.subject_data[subject_category_name][data[subject_category_name]]
+                        add_subject_assignments(policy_id, subject_id, subject_cat_id, subject_data_id)
+        else:
+            for subject_category_name in scenario.subject_assignments[subject_name]:
+                subject_id = scenario.subjects[subject_name]
+                subject_cat_id = scenario.subject_categories[subject_category_name]
+                subject_data_id = scenario.subject_data[subject_category_name][scenario.subject_assignments[subject_name][subject_category_name]]
+                add_subject_assignments(policy_id, subject_id, subject_cat_id, subject_data_id)
+        
+    logger.info("Add object assignments")
+    for object_name in scenario.object_assignments:
+        if type(scenario.object_assignments[object_name]) in (list, tuple):
+            for items in scenario.object_assignments[object_name]:
+                for object_category_name in items:
+                    object_id = scenario.objects[object_name]
+                    object_cat_id = scenario.object_categories[object_category_name]
+                    for data in scenario.object_assignments[object_name]:
+                        object_data_id = scenario.object_data[object_category_name][data[object_category_name]]
+                        add_object_assignments(policy_id, object_id, object_cat_id, object_data_id)
+        else:
+            for object_category_name in scenario.object_assignments[object_name]:
+                object_id = scenario.objects[object_name]
+                object_cat_id = scenario.object_categories[object_category_name]
+                object_data_id = scenario.object_data[object_category_name][scenario.object_assignments[object_name][object_category_name]]
+                add_object_assignments(policy_id, object_id, object_cat_id, object_data_id)
+
+    logger.info("Add action assignments")
+    for action_name in scenario.action_assignments:
+        if type(scenario.action_assignments[action_name]) in (list, tuple):
+            for items in scenario.action_assignments[action_name]:
+                for action_category_name in items:
+                    action_id = scenario.actions[action_name]
+                    action_cat_id = scenario.action_categories[action_category_name]
+                    for data in scenario.action_assignments[action_name]:
+                        action_data_id = scenario.action_data[action_category_name][data[action_category_name]]
+                        add_action_assignments(policy_id, action_id, action_cat_id, action_data_id)
+        else:
+            for action_category_name in scenario.action_assignments[action_name]:
+                action_id = scenario.actions[action_name]
+                action_cat_id = scenario.action_categories[action_category_name]
+                action_data_id = scenario.action_data[action_category_name][scenario.action_assignments[action_name][action_category_name]]
+                add_action_assignments(policy_id, action_id, action_cat_id, action_data_id)
+
+    logger.info("Add rules")
+    for meta_rule_name in scenario.rules:
+        meta_rule_value = scenario.meta_rule[meta_rule_name]
+        for rule in scenario.rules[meta_rule_name]:
+            data_list = []
+            _meta_rule = list(meta_rule_value["value"])
+            for data_name in rule["rule"]:
+                category_name = _meta_rule.pop(0)
+                if category_name in scenario.subject_categories:
+                    data_list.append(scenario.subject_data[category_name][data_name])
+                elif category_name in scenario.object_categories:
+                    data_list.append(scenario.object_data[category_name][data_name])
+                elif category_name in scenario.action_categories:
+                    data_list.append(scenario.action_data[category_name][data_name])
+            instructions = rule["instructions"]
+            add_rule(policy_id, meta_rule_value["id"], data_list, instructions)
+    return policy_id
+
+
+def create_pdp(policy_id=None):
+    logger.info("Creating PDP {}".format(scenario.pdp_name))
+    projects = get_keystone_projects()
+    project_id = args.keystone_pid
+    if not project_id:
+        for _project in projects['projects']:
+            if _project['name'] == "admin":
+                project_id = _project['id']
+    assert project_id
+    pdps = check_pdp()["pdps"]
+    for pdp_id, pdp_value in pdps.items():
+        if scenario.pdp_name == pdp_value["name"]:
+            update_pdp(pdp_id, policy_id=policy_id)
+            logger.debug("Found existing PDP named {} (will add policy {})".format(scenario.pdp_name, policy_id))
+            return pdp_id
+    _pdp_id = add_pdp(name=scenario.pdp_name, policy_id=policy_id)
+    map_to_keystone(pdp_id=_pdp_id, keystone_project_id=project_id)
+    return _pdp_id
+
+if __name__ == "__main__":
+    _models = check_model()
+    for _model_id, _model_value in _models['models'].items():
+        if _model_value['name'] == scenario.model_name:
+            model_id = _model_id
+            meta_rule_list = _model_value['meta_rules']
+            create_model(model_id)
+            break
+    else:
+        model_id, meta_rule_list = create_model()
+    policy_id = create_policy(model_id, meta_rule_list)
+    pdp_id = create_pdp(policy_id)
diff --git a/moon_forming/run.sh b/moon_forming/run.sh
new file mode 100644
index 00000000..71543f9e
--- /dev/null
+++ b/moon_forming/run.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+populate_args=$*
+
+echo "Waiting for Consul (http://consul:8500)"
+while ! python -c "import requests; req = requests.get('http://consul:8500')" 2>/dev/null ; do
+    sleep 5 ;
+    echo "."
+done
+
+echo "Consul (http://consul:8500) is up."
+
+python3 /root/conf2consul.py /etc/moon/moon.conf
+
+echo "Waiting for DB (tcp://db:3306)"
+while ! python -c "import socket, sys; s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.connect(('db', 3306)); sys.exit(0)" 2>/dev/null ; do
+    sleep 5 ;
+    echo "."
+done
+
+echo "Database (http://db:3306) is up."
+
+moon_db_manager upgrade
+
+echo "Waiting for Keystone (http://keystone:5000)"
+while ! python -c "import requests; req = requests.get('http://keystone:5000')" 2>/dev/null ; do
+    sleep 5 ;
+    echo "."
+done
+
+echo "Keystone (http://keystone:5000) is up."
+
+echo "Waiting for Manager (http://manager:8082)"
+while ! python -c "import requests; req = requests.get('http://manager:8082')" 2>/dev/null ; do
+    sleep 5 ;
+    echo "."
+done
+
+echo "Manager (http://manager:8082) is up."
+
+cd /root
+
+python3 populate_default_values.py $populate_args /root/conf/rbac.py
+python3 populate_default_values.py $populate_args /root/conf/mls.py
diff --git a/moon_forming/utils/__init__.py b/moon_forming/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/moon_forming/utils/__init__.py
diff --git a/moon_forming/utils/config.py b/moon_forming/utils/config.py
new file mode 100644
index 00000000..30c8ea4f
--- /dev/null
+++ b/moon_forming/utils/config.py
@@ -0,0 +1,22 @@
+import yaml
+
+
+def get_config_data(filename="moon.conf"):
+    data_config = None
+    for _file in (
+            filename,
+            "conf/moon.conf",
+            "../moon.conf",
+            "../conf/moon.conf",
+            "/etc/moon/moon.conf",
+    ):
+        try:
+            data_config = yaml.safe_load(open(_file))
+        except FileNotFoundError:
+            data_config = None
+            continue
+        else:
+            break
+    if not data_config:
+        raise Exception("Configuration file not found...")
+    return data_config
diff --git a/moon_forming/utils/models.py b/moon_forming/utils/models.py
new file mode 100644
index 00000000..3cf31354
--- /dev/null
+++ b/moon_forming/utils/models.py
@@ -0,0 +1,270 @@
+import requests
+import copy
+import utils.config
+
+config = utils.config.get_config_data()
+
+URL = "http://{}:{}".format(
+    config['components']['manager']['hostname'],
+    config['components']['manager']['port'])
+URL = URL + "{}"
+HEADERS = {"content-type": "application/json"}
+
+model_template = {
+    "name": "test_model",
+    "description": "test",
+    "meta_rules": []
+}
+
+category_template = {
+    "name": "name of the category",
+    "description": "description of the category"
+}
+
+meta_rule_template = {
+    "name": "test_meta_rule",
+    "subject_categories": [],
+    "object_categories": [],
+    "action_categories": []
+}
+
+
+def check_model(model_id=None, check_model_name=True):
+    req = requests.get(URL.format("/models"))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "models" in result
+    if model_id:
+        assert result["models"]
+        assert model_id in result['models']
+        assert "name" in result['models'][model_id]
+        if check_model_name:
+            assert model_template["name"] == result['models'][model_id]["name"]
+    return result
+
+
+def add_model(name=None):
+    if name:
+        model_template['name'] = name
+    req = requests.post(URL.format("/models"), json=model_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    model_id = list(result['models'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "name" in result['models'][model_id]
+    assert model_template["name"] == result['models'][model_id]["name"]
+    return model_id
+
+
+def delete_model(model_id):
+    req = requests.delete(URL.format("/models/{}".format(model_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "result" in result
+    assert result["result"]
+
+
+def add_subject_category(name="subject_cat_1"):
+    category_template["name"] = name
+    req = requests.post(URL.format("/subject_categories"), json=category_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "subject_categories" in result
+    category_id = list(result['subject_categories'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "name" in result['subject_categories'][category_id]
+    assert category_template["name"] == result['subject_categories'][category_id]["name"]
+    return category_id
+
+
+def check_subject_category(category_id):
+    req = requests.get(URL.format("/subject_categories"))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "subject_categories" in result
+    if "result" in result:
+        assert result["result"]
+    assert category_id in result['subject_categories']
+    assert "name" in result['subject_categories'][category_id]
+    assert category_template["name"] == result['subject_categories'][category_id]["name"]
+
+
+def delete_subject_category(category_id):
+    req = requests.delete(URL.format("/subject_categories/{}".format(category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    if "result" in result:
+        assert result["result"]
+
+
+def add_object_category(name="object_cat_1"):
+    category_template["name"] = name
+    req = requests.post(URL.format("/object_categories"), json=category_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "object_categories" in result
+    category_id = list(result['object_categories'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "name" in result['object_categories'][category_id]
+    assert category_template["name"] == result['object_categories'][category_id]["name"]
+    return category_id
+
+
+def check_object_category(category_id):
+    req = requests.get(URL.format("/object_categories"))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "object_categories" in result
+    if "result" in result:
+        assert result["result"]
+    assert category_id in result['object_categories']
+    assert "name" in result['object_categories'][category_id]
+    assert category_template["name"] == result['object_categories'][category_id]["name"]
+
+
+def delete_object_category(category_id):
+    req = requests.delete(URL.format("/object_categories/{}".format(category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    if "result" in result:
+        assert result["result"]
+
+
+def add_action_category(name="action_cat_1"):
+    category_template["name"] = name
+    req = requests.post(URL.format("/action_categories"), json=category_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "action_categories" in result
+    category_id = list(result['action_categories'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "name" in result['action_categories'][category_id]
+    assert category_template["name"] == result['action_categories'][category_id]["name"]
+    return category_id
+
+
+def check_action_category(category_id):
+    req = requests.get(URL.format("/action_categories"))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "action_categories" in result
+    if "result" in result:
+        assert result["result"]
+    assert category_id in result['action_categories']
+    assert "name" in result['action_categories'][category_id]
+    assert category_template["name"] == result['action_categories'][category_id]["name"]
+
+
+def delete_action_category(category_id):
+    req = requests.delete(URL.format("/action_categories/{}".format(category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    if "result" in result:
+        assert result["result"]
+
+
+def add_categories_and_meta_rule(name="test_meta_rule"):
+    scat_id = add_subject_category()
+    ocat_id = add_object_category()
+    acat_id = add_action_category()
+    _meta_rule_template = copy.deepcopy(meta_rule_template)
+    _meta_rule_template["name"] = name
+    _meta_rule_template["subject_categories"].append(scat_id)
+    _meta_rule_template["object_categories"].append(ocat_id)
+    _meta_rule_template["action_categories"].append(acat_id)
+    req = requests.post(URL.format("/meta_rules"), json=_meta_rule_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "meta_rules" in result
+    meta_rule_id = list(result['meta_rules'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "name" in result['meta_rules'][meta_rule_id]
+    assert _meta_rule_template["name"] == result['meta_rules'][meta_rule_id]["name"]
+    return meta_rule_id, scat_id, ocat_id, acat_id
+
+
+def add_meta_rule(name="test_meta_rule", scat=[], ocat=[], acat=[]):
+    _meta_rule_template = copy.deepcopy(meta_rule_template)
+    _meta_rule_template["name"] = name
+    _meta_rule_template["subject_categories"] = []
+    _meta_rule_template["subject_categories"].extend(scat)
+    _meta_rule_template["object_categories"] = []
+    _meta_rule_template["object_categories"].extend(ocat)
+    _meta_rule_template["action_categories"] = []
+    _meta_rule_template["action_categories"].extend(acat)
+    req = requests.post(URL.format("/meta_rules"), json=_meta_rule_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "meta_rules" in result
+    meta_rule_id = list(result['meta_rules'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "name" in result['meta_rules'][meta_rule_id]
+    assert _meta_rule_template["name"] == result['meta_rules'][meta_rule_id]["name"]
+    return meta_rule_id
+
+
+def check_meta_rule(meta_rule_id, scat_id=None, ocat_id=None, acat_id=None):
+    req = requests.get(URL.format("/meta_rules"))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "meta_rules" in result
+    if "result" in result:
+        assert result["result"]
+    if not meta_rule_id:
+        return result
+    assert meta_rule_id in result['meta_rules']
+    assert "name" in result['meta_rules'][meta_rule_id]
+    if scat_id:
+        assert scat_id in result['meta_rules'][meta_rule_id]["subject_categories"]
+    if ocat_id:
+        assert ocat_id in result['meta_rules'][meta_rule_id]["object_categories"]
+    if acat_id:
+        assert acat_id in result['meta_rules'][meta_rule_id]["action_categories"]
+
+
+def delete_meta_rule(meta_rule_id):
+    req = requests.delete(URL.format("/meta_rules/{}".format(meta_rule_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    if "result" in result:
+        assert result["result"]
+
+
+def add_meta_rule_to_model(model_id, meta_rule_id):
+    model = check_model(model_id, check_model_name=False)['models']
+    meta_rule_list = model[model_id]["meta_rules"]
+    if meta_rule_id not in meta_rule_list:
+        meta_rule_list.append(meta_rule_id)
+        req = requests.patch(URL.format("/models/{}".format(model_id)),
+                             json={"meta_rules": meta_rule_list},
+                             headers=HEADERS)
+        assert req.status_code == 200
+        result = req.json()
+        assert type(result) is dict
+        model_id = list(result['models'].keys())[0]
+        if "result" in result:
+            assert result["result"]
+        assert "meta_rules" in result['models'][model_id]
+        assert meta_rule_list == result['models'][model_id]["meta_rules"]
diff --git a/moon_forming/utils/pdp.py b/moon_forming/utils/pdp.py
new file mode 100644
index 00000000..f3c6df37
--- /dev/null
+++ b/moon_forming/utils/pdp.py
@@ -0,0 +1,163 @@
+import logging
+import requests
+import utils.config
+
+config = utils.config.get_config_data()
+logger = logging.getLogger("moonforming.utils.policies")
+
+URL = "http://{}:{}".format(
+    config['components']['manager']['hostname'],
+    config['components']['manager']['port'])
+HEADERS = {"content-type": "application/json"}
+KEYSTONE_USER = config['openstack']['keystone']['user']
+KEYSTONE_PASSWORD = config['openstack']['keystone']['password']
+KEYSTONE_PROJECT = config['openstack']['keystone']['project']
+KEYSTONE_SERVER = config['openstack']['keystone']['url']
+
+pdp_template = {
+    "name": "test_pdp",
+    "security_pipeline": [],
+    "keystone_project_id": None,
+    "description": "test",
+}
+
+
+def get_keystone_projects():
+
+    HEADERS = {
+        "Content-Type": "application/json"
+    }
+
+    data_auth = {
+        "auth": {
+            "identity": {
+                "methods": [
+                    "password"
+                ],
+                "password": {
+                    "user": {
+                        "name": KEYSTONE_USER,
+                        "domain": {
+                            "name": "Default"
+                        },
+                        "password": KEYSTONE_PASSWORD
+                    }
+                }
+            }
+        }
+    }
+
+    req = requests.post("{}/auth/tokens".format(KEYSTONE_SERVER), json=data_auth, headers=HEADERS)
+    logger.debug("{}/auth/tokens".format(KEYSTONE_SERVER))
+    logger.debug(req.text)
+    assert req.status_code in (200, 201)
+    TOKEN = req.headers['X-Subject-Token']
+    HEADERS['X-Auth-Token'] = TOKEN
+    req = requests.get("{}/projects".format(KEYSTONE_SERVER), headers=HEADERS)
+    if req.status_code not in (200, 201):
+        data_auth["auth"]["scope"] = {
+            "project": {
+                "name": KEYSTONE_PROJECT,
+                "domain": {
+                    "id": "default"
+                }
+            }
+        }
+        req = requests.post("{}/auth/tokens".format(KEYSTONE_SERVER), json=data_auth, headers=HEADERS)
+        assert req.status_code in (200, 201)
+        TOKEN = req.headers['X-Subject-Token']
+        HEADERS['X-Auth-Token'] = TOKEN
+        req = requests.get("{}/projects".format(KEYSTONE_SERVER), headers=HEADERS)
+    assert req.status_code in (200, 201)
+    return req.json()
+
+
+def check_pdp(pdp_id=None, keystone_project_id=None, moon_url=None):
+    _URL = URL
+    if moon_url:
+        _URL = moon_url
+    req = requests.get(_URL + "/pdp")
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "pdps" in result
+    if pdp_id:
+        assert result["pdps"]
+        assert pdp_id in result['pdps']
+        assert "name" in result['pdps'][pdp_id]
+        assert pdp_template["name"] == result['pdps'][pdp_id]["name"]
+    if keystone_project_id:
+        assert result["pdps"]
+        assert pdp_id in result['pdps']
+        assert "keystone_project_id" in result['pdps'][pdp_id]
+        assert keystone_project_id == result['pdps'][pdp_id]["keystone_project_id"]
+    return result
+
+
+def add_pdp(name="test_pdp", policy_id=None):
+    pdp_template['name'] = name
+    if policy_id:
+        pdp_template['security_pipeline'].append(policy_id)
+    req = requests.post(URL + "/pdp", json=pdp_template, headers=HEADERS)
+    logger.debug(req.status_code)
+    logger.debug(req)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    pdp_id = list(result['pdps'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "name" in result['pdps'][pdp_id]
+    assert pdp_template["name"] == result['pdps'][pdp_id]["name"]
+    return pdp_id
+
+
+def update_pdp(pdp_id, policy_id=None):
+    req = requests.get(URL + "/pdp/{}".format(pdp_id))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "pdps" in result
+    assert pdp_id in result['pdps']
+    pipeline = result['pdps'][pdp_id]["security_pipeline"]
+    if policy_id not in pipeline:
+        pipeline.append(policy_id)
+        req = requests.patch(URL + "/pdp/{}".format(pdp_id),
+                             json={"security_pipeline": pipeline})
+        assert req.status_code == 200
+        result = req.json()
+        assert type(result) is dict
+        assert "pdps" in result
+        assert pdp_id in result['pdps']
+
+    req = requests.get(URL + "/pdp/{}".format(pdp_id))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "pdps" in result
+    assert pdp_id in result['pdps']
+    assert policy_id in pipeline
+
+
+def map_to_keystone(pdp_id, keystone_project_id):
+    req = requests.patch(URL + "/pdp/{}".format(pdp_id), json={"keystone_project_id": keystone_project_id},
+                         headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    if "result" in result:
+        assert result["result"]
+    assert pdp_id in result['pdps']
+    assert "name" in result['pdps'][pdp_id]
+    assert pdp_template["name"] == result['pdps'][pdp_id]["name"]
+    return pdp_id
+
+
+def delete_pdp(pdp_id):
+    req = requests.delete(URL + "/pdp/{}".format(pdp_id))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "result" in result
+    assert result["result"]
+
diff --git a/moon_forming/utils/policies.py b/moon_forming/utils/policies.py
new file mode 100644
index 00000000..bd08291a
--- /dev/null
+++ b/moon_forming/utils/policies.py
@@ -0,0 +1,635 @@
+import logging
+import requests
+import utils.config
+
+config = utils.config.get_config_data()
+logger = logging.getLogger("moonforming.utils.policies")
+
+URL = "http://{}:{}".format(config['components']['manager']['hostname'], config['components']['manager']['port'])
+URL = URL + "{}"
+HEADERS = {"content-type": "application/json"}
+FILE = open("/tmp/test.log", "w")
+
+policy_template = {
+    "name": "test_policy",
+    "model_id": "",
+    "genre": "authz",
+    "description": "test",
+}
+
+subject_template = {
+    "name": "test_subject",
+    "description": "test",
+    "email": "mail",
+    "password": "my_pass",
+}
+
+object_template = {
+    "name": "test_subject",
+    "description": "test"
+}
+
+action_template = {
+    "name": "test_subject",
+    "description": "test"
+}
+
+subject_data_template = {
+    "name": "subject_data1",
+    "description": "description of the data subject"
+}
+
+object_data_template = {
+    "name": "object_data1",
+    "description": "description of the data subject"
+}
+
+action_data_template = {
+    "name": "action_data1",
+    "description": "description of the data subject"
+}
+
+subject_assignment_template = {
+    "id": "",
+    "category_id": "",
+    "scope_id": ""
+}
+
+
+def check_policy(policy_id=None):
+    req = requests.get(URL.format("/policies"))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "policies" in result
+    if policy_id:
+        assert result["policies"]
+        assert policy_id in result['policies']
+        assert "name" in result['policies'][policy_id]
+        assert policy_template["name"] == result['policies'][policy_id]["name"]
+    return result
+
+
+def add_policy(name="test_policy", genre="authz"):
+    policy_template["name"] = name
+    policy_template["genre"] = genre
+    req = requests.post(URL.format("/policies"), json=policy_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    policy_id = list(result['policies'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "name" in result['policies'][policy_id]
+    assert policy_template["name"] == result['policies'][policy_id]["name"]
+    return policy_id
+
+
+def update_policy(policy_id, model_id):
+    req = requests.patch(URL.format("/policies/{}".format(policy_id)),
+                         json={"model_id": model_id}, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    policy_id = list(result['policies'].keys())[0]
+    if "result" in result:
+        assert result["result"]
+    assert "model_id" in result['policies'][policy_id]
+    assert model_id == result['policies'][policy_id]["model_id"]
+
+
+def delete_policy(policy_id):
+    req = requests.delete(URL.format("/policies/{}".format(policy_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "result" in result
+    assert result["result"]
+
+
+def add_subject(policy_id=None, name="test_subject"):
+    subject_template['name'] = name
+    if policy_id:
+        logger.debug(URL.format("/policies/{}/subjects".format(policy_id)))
+        req = requests.post(URL.format("/policies/{}/subjects".format(policy_id)),
+                            json=subject_template, headers=HEADERS)
+    else:
+        logger.debug(URL.format("/subjects"))
+        req = requests.post(URL.format("/subjects"), json=subject_template, headers=HEADERS)
+    logger.debug(req.text)
+    assert req.status_code == 200
+    result = req.json()
+    assert "subjects" in result
+    subject_id = list(result['subjects'].keys())[0]
+    return subject_id
+
+
+def update_subject(subject_id, policy_id=None, description=None):
+    if policy_id and not description:
+        req = requests.patch(URL.format("/policies/{}/subjects/{}".format(policy_id, subject_id)),
+                             json={})
+    elif policy_id and description:
+        req = requests.patch(URL.format("/policies/{}/subjects/{}".format(policy_id, subject_id)),
+                             json={"description": description})
+    else:
+        req = requests.patch(URL.format("/subjects/{}".format(subject_id)),
+                             json={"description": description})
+    assert req.status_code == 200
+    result = req.json()
+    assert "subjects" in result
+    assert "name" in result["subjects"][subject_id]
+    assert subject_template["name"] == result["subjects"][subject_id]["name"]
+    assert "policy_list" in result["subjects"][subject_id]
+    if policy_id:
+        assert policy_id in result["subjects"][subject_id]["policy_list"]
+    if description:
+        assert description in result["subjects"][subject_id]["description"]
+
+
+def check_subject(subject_id=None, policy_id=None):
+    if policy_id:
+        req = requests.get(URL.format("/policies/{}/subjects".format(policy_id)))
+    else:
+        req = requests.get(URL.format("/subjects"))
+    assert req.status_code == 200
+    result = req.json()
+    assert "subjects" in result
+    assert "name" in result["subjects"][subject_id]
+    assert subject_template["name"] == result["subjects"][subject_id]["name"]
+    if policy_id:
+        assert "policy_list" in result["subjects"][subject_id]
+        assert policy_id in result["subjects"][subject_id]["policy_list"]
+
+
+def delete_subject(subject_id, policy_id=None):
+    if policy_id:
+        req = requests.delete(URL.format("/policies/{}/subjects/{}".format(policy_id, subject_id)))
+    else:
+        req = requests.delete(URL.format("/subjects/{}".format(subject_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "result" in result
+    assert result["result"]
+
+    if policy_id:
+        req = requests.get(URL.format("/policies/{}/subjects".format(policy_id)))
+    else:
+        req = requests.get(URL.format("/subjects"))
+    assert req.status_code == 200
+    result = req.json()
+    assert "subjects" in result
+    if subject_id in result["subjects"]:
+        assert "name" in result["subjects"][subject_id]
+        assert subject_template["name"] == result["subjects"][subject_id]["name"]
+        if policy_id:
+            assert "policy_list" in result["subjects"][subject_id]
+            assert policy_id not in result["subjects"][subject_id]["policy_list"]
+
+
+def add_object(policy_id=None, name="test_object"):
+    object_template['name'] = name
+    if policy_id:
+        req = requests.post(URL.format("/policies/{}/objects".format(policy_id)),
+                            json=object_template, headers=HEADERS)
+    else:
+        req = requests.post(URL.format("/objects"), json=object_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "objects" in result
+    object_id = list(result['objects'].keys())[0]
+    return object_id
+
+
+def update_object(object_id, policy_id):
+    req = requests.patch(URL.format("/policies/{}/objects/{}".format(policy_id, object_id)), json={})
+    assert req.status_code == 200
+    result = req.json()
+    assert "objects" in result
+    assert "name" in result["objects"][object_id]
+    assert object_template["name"] == result["objects"][object_id]["name"]
+    assert "policy_list" in result["objects"][object_id]
+    assert policy_id in result["objects"][object_id]["policy_list"]
+
+
+def check_object(object_id=None, policy_id=None):
+    if policy_id:
+        req = requests.get(URL.format("/policies/{}/objects".format(policy_id)))
+    else:
+        req = requests.get(URL.format("/objects"))
+    assert req.status_code == 200
+    result = req.json()
+    assert "objects" in result
+    assert "name" in result["objects"][object_id]
+    assert object_template["name"] == result["objects"][object_id]["name"]
+    if policy_id:
+        assert "policy_list" in result["objects"][object_id]
+        assert policy_id in result["objects"][object_id]["policy_list"]
+
+
+def delete_object(object_id, policy_id=None):
+    if policy_id:
+        req = requests.delete(URL.format("/policies/{}/objects/{}".format(policy_id, object_id)))
+    else:
+        req = requests.delete(URL.format("/objects/{}".format(object_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "result" in result
+    assert result["result"]
+
+    if policy_id:
+        req = requests.get(URL.format("/policies/{}/objects".format(policy_id)))
+    else:
+        req = requests.get(URL.format("/objects"))
+    assert req.status_code == 200
+    result = req.json()
+    assert "objects" in result
+    if object_id in result["objects"]:
+        assert "name" in result["objects"][object_id]
+        assert object_template["name"] == result["objects"][object_id]["name"]
+        if policy_id:
+            assert "policy_list" in result["objects"][object_id]
+            assert policy_id not in result["objects"][object_id]["policy_list"]
+
+
+def add_action(policy_id=None, name="test_action"):
+    action_template['name'] = name
+    if policy_id:
+        req = requests.post(URL.format("/policies/{}/actions".format(policy_id)),
+                            json=action_template, headers=HEADERS)
+    else:
+        req = requests.post(URL.format("/actions"), json=action_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "actions" in result
+    action_id = list(result['actions'].keys())[0]
+    return action_id
+
+
+def update_action(action_id, policy_id):
+    req = requests.patch(URL.format("/policies/{}/actions/{}".format(policy_id, action_id)), json={})
+    assert req.status_code == 200
+    result = req.json()
+    assert "actions" in result
+    assert "name" in result["actions"][action_id]
+    assert action_template["name"] == result["actions"][action_id]["name"]
+    assert "policy_list" in result["actions"][action_id]
+    assert policy_id in result["actions"][action_id]["policy_list"]
+
+
+def check_action(action_id=None, policy_id=None):
+    if policy_id:
+        req = requests.get(URL.format("/policies/{}/actions".format(policy_id)))
+    else:
+        req = requests.get(URL.format("/actions"))
+    assert req.status_code == 200
+    result = req.json()
+    assert "actions" in result
+    assert "name" in result["actions"][action_id]
+    assert action_template["name"] == result["actions"][action_id]["name"]
+    if policy_id:
+        assert "policy_list" in result["actions"][action_id]
+        assert policy_id in result["actions"][action_id]["policy_list"]
+
+
+def delete_action(action_id, policy_id=None):
+    if policy_id:
+        req = requests.delete(URL.format("/policies/{}/actions/{}".format(policy_id, action_id)))
+    else:
+        req = requests.delete(URL.format("/actions/{}".format(action_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert type(result) is dict
+    assert "result" in result
+    assert result["result"]
+
+    if policy_id:
+        req = requests.get(URL.format("/policies/{}/actions".format(policy_id)))
+    else:
+        req = requests.get(URL.format("/actions"))
+    assert req.status_code == 200
+    result = req.json()
+    assert "actions" in result
+    if action_id in result["actions"]:
+        assert "name" in result["actions"][action_id]
+        assert action_template["name"] == result["actions"][action_id]["name"]
+        if policy_id:
+            assert "policy_list" in result["actions"][action_id]
+            assert policy_id not in result["actions"][action_id]["policy_list"]
+
+
+def add_subject_data(policy_id, category_id, name="subject_data1"):
+    subject_data_template['name'] = name
+    req = requests.post(URL.format("/policies/{}/subject_data/{}".format(policy_id, category_id)),
+                        json=subject_data_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "subject_data" in result
+    subject_id = list(result['subject_data']['data'].keys())[0]
+    return subject_id
+
+
+def check_subject_data(policy_id, data_id, category_id):
+    req = requests.get(URL.format("/policies/{}/subject_data/{}".format(policy_id, category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "subject_data" in result
+    for _data in result['subject_data']:
+        assert data_id in list(_data['data'].keys())
+        assert category_id == _data["category_id"]
+
+
+def delete_subject_data(policy_id, category_id, data_id):
+    req = requests.delete(URL.format("/policies/{}/subject_data/{}/{}".format(policy_id, category_id, data_id)),
+                          headers=HEADERS)
+    assert req.status_code == 200
+    req = requests.get(URL.format("/policies/{}/subject_data/{}".format(policy_id, category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "subject_data" in result
+    for _data in result['subject_data']:
+        assert data_id not in list(_data['data'].keys())
+        assert category_id == _data["category_id"]
+
+
+def add_object_data(policy_id, category_id, name="object_data1"):
+    object_data_template['name'] = name
+    req = requests.post(URL.format("/policies/{}/object_data/{}".format(policy_id, category_id)),
+                        json=object_data_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "object_data" in result
+    object_id = list(result['object_data']['data'].keys())[0]
+    return object_id
+
+
+def check_object_data(policy_id, data_id, category_id):
+    req = requests.get(URL.format("/policies/{}/object_data/{}".format(policy_id, category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "object_data" in result
+    for _data in result['object_data']:
+        assert data_id in list(_data['data'].keys())
+        assert category_id == _data["category_id"]
+
+
+def delete_object_data(policy_id, category_id, data_id):
+    req = requests.delete(URL.format("/policies/{}/object_data/{}/{}".format(policy_id, category_id, data_id)),
+                          headers=HEADERS)
+    assert req.status_code == 200
+    req = requests.get(URL.format("/policies/{}/object_data/{}".format(policy_id, category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "object_data" in result
+    for _data in result['object_data']:
+        assert data_id not in list(_data['data'].keys())
+        assert category_id == _data["category_id"]
+
+
+def add_action_data(policy_id, category_id, name="action_data1"):
+    action_data_template['name'] = name
+    req = requests.post(URL.format("/policies/{}/action_data/{}".format(policy_id, category_id)),
+                        json=action_data_template, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "action_data" in result
+    action_id = list(result['action_data']['data'].keys())[0]
+    return action_id
+
+
+def check_action_data(policy_id, data_id, category_id):
+    req = requests.get(URL.format("/policies/{}/action_data/{}".format(policy_id, category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "action_data" in result
+    for _data in result['action_data']:
+        assert data_id in list(_data['data'].keys())
+        assert category_id == _data["category_id"]
+
+
+def delete_action_data(policy_id, category_id, data_id):
+    req = requests.delete(URL.format("/policies/{}/action_data/{}/{}".format(policy_id, category_id, data_id)),
+                          headers=HEADERS)
+    assert req.status_code == 200
+    req = requests.get(URL.format("/policies/{}/action_data/{}".format(policy_id, category_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "action_data" in result
+    for _data in result['action_data']:
+        assert data_id not in list(_data['data'].keys())
+        assert category_id == _data["category_id"]
+
+
+def add_subject_assignments(policy_id, subject_id, subject_cat_id, subject_data_id):
+    req = requests.post(URL.format("/policies/{}/subject_assignments".format(policy_id)),
+                        json={
+                                "id": subject_id,
+                                "category_id": subject_cat_id,
+                                "data_id": subject_data_id
+                            }, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "subject_assignments" in result
+    assert result["subject_assignments"]
+
+
+def check_subject_assignments(policy_id, subject_id, subject_cat_id, subject_data_id):
+    req = requests.get(URL.format("/policies/{}/subject_assignments/{}/{}/{}".format(
+        policy_id, subject_id, subject_cat_id, subject_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "subject_assignments" in result
+    assert result["subject_assignments"]
+    for key in result["subject_assignments"]:
+        assert "subject_id" in result["subject_assignments"][key]
+        assert "category_id" in result["subject_assignments"][key]
+        assert "assignments" in result["subject_assignments"][key]
+        if result["subject_assignments"][key]['subject_id'] == subject_id and \
+                result["subject_assignments"][key]["category_id"] == subject_cat_id:
+            assert subject_data_id in result["subject_assignments"][key]["assignments"]
+
+
+def check_object_assignments(policy_id, object_id, object_cat_id, object_data_id):
+    req = requests.get(URL.format("/policies/{}/object_assignments/{}/{}/{}".format(
+        policy_id, object_id, object_cat_id, object_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "object_assignments" in result
+    assert result["object_assignments"]
+    for key in result["object_assignments"]:
+        assert "object_id" in result["object_assignments"][key]
+        assert "category_id" in result["object_assignments"][key]
+        assert "assignments" in result["object_assignments"][key]
+        if result["object_assignments"][key]['object_id'] == object_id and \
+                result["object_assignments"][key]["category_id"] == object_cat_id:
+            assert object_data_id in result["object_assignments"][key]["assignments"]
+
+
+def check_action_assignments(policy_id, action_id, action_cat_id, action_data_id):
+    req = requests.get(URL.format("/policies/{}/action_assignments/{}/{}/{}".format(
+        policy_id, action_id, action_cat_id, action_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "action_assignments" in result
+    assert result["action_assignments"]
+    for key in result["action_assignments"]:
+        assert "action_id" in result["action_assignments"][key]
+        assert "category_id" in result["action_assignments"][key]
+        assert "assignments" in result["action_assignments"][key]
+        if result["action_assignments"][key]['action_id'] == action_id and \
+                result["action_assignments"][key]["category_id"] == action_cat_id:
+            assert action_data_id in result["action_assignments"][key]["assignments"]
+
+
+def add_object_assignments(policy_id, object_id, object_cat_id, object_data_id):
+    req = requests.post(URL.format("/policies/{}/object_assignments".format(policy_id)),
+                        json={
+                                "id": object_id,
+                                "category_id": object_cat_id,
+                                "data_id": object_data_id
+                            }, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "object_assignments" in result
+    assert result["object_assignments"]
+
+
+def add_action_assignments(policy_id, action_id, action_cat_id, action_data_id):
+    req = requests.post(URL.format("/policies/{}/action_assignments".format(policy_id)),
+                        json={
+                                "id": action_id,
+                                "category_id": action_cat_id,
+                                "data_id": action_data_id
+                            }, headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "action_assignments" in result
+    assert result["action_assignments"]
+
+
+def delete_subject_assignment(policy_id, subject_id, subject_cat_id, subject_data_id):
+    req = requests.delete(URL.format("/policies/{}/subject_assignments/{}/{}/{}".format(
+        policy_id, subject_id, subject_cat_id, subject_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "result" in result
+    assert result["result"]
+
+    req = requests.get(URL.format("/policies/{}/subject_assignments/{}/{}/{}".format(
+        policy_id, subject_id, subject_cat_id, subject_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "subject_assignments" in result
+    assert result["subject_assignments"]
+    for key in result["subject_assignments"]:
+        assert "subject_id" in result["subject_assignments"][key]
+        assert "category_id" in result["subject_assignments"][key]
+        assert "assignments" in result["subject_assignments"][key]
+        if result["subject_assignments"][key]['subject_id'] == subject_id and \
+                result["subject_assignments"][key]["category_id"] == subject_cat_id:
+            assert subject_data_id not in result["subject_assignments"][key]["assignments"]
+
+
+def delete_object_assignment(policy_id, object_id, object_cat_id, object_data_id):
+    req = requests.delete(URL.format("/policies/{}/object_assignments/{}/{}/{}".format(
+        policy_id, object_id, object_cat_id, object_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "result" in result
+    assert result["result"]
+
+    req = requests.get(URL.format("/policies/{}/object_assignments/{}/{}/{}".format(
+        policy_id, object_id, object_cat_id, object_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "object_assignments" in result
+    assert result["object_assignments"]
+    for key in result["object_assignments"]:
+        assert "object_id" in result["object_assignments"][key]
+        assert "category_id" in result["object_assignments"][key]
+        assert "assignments" in result["object_assignments"][key]
+        if result["object_assignments"][key]['object_id'] == object_id and \
+                result["object_assignments"][key]["category_id"] == object_cat_id:
+            assert object_data_id not in result["object_assignments"][key]["assignments"]
+
+
+def delete_action_assignment(policy_id, action_id, action_cat_id, action_data_id):
+    req = requests.delete(URL.format("/policies/{}/action_assignments/{}/{}/{}".format(
+        policy_id, action_id, action_cat_id, action_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "result" in result
+    assert result["result"]
+
+    req = requests.get(URL.format("/policies/{}/action_assignments/{}/{}/{}".format(
+        policy_id, action_id, action_cat_id, action_data_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "action_assignments" in result
+    assert result["action_assignments"]
+    for key in result["action_assignments"]:
+        assert "action_id" in result["action_assignments"][key]
+        assert "category_id" in result["action_assignments"][key]
+        assert "assignments" in result["action_assignments"][key]
+        if result["action_assignments"][key]['action_id'] == action_id and \
+                result["action_assignments"][key]["category_id"] == action_cat_id:
+            assert action_data_id not in result["action_assignments"][key]["assignments"]
+
+
+def add_rule(policy_id, meta_rule_id, rule, instructions={"chain": [{"security_pipeline": "rbac"}]}):
+    req = requests.post(URL.format("/policies/{}/rules".format(policy_id)),
+                        json={
+                            "meta_rule_id": meta_rule_id,
+                            "rule": rule,
+                            "instructions": instructions,
+                            "enabled": True
+                        },
+                        headers=HEADERS)
+    assert req.status_code == 200
+    result = req.json()
+    assert "rules" in result
+    try:
+        rule_id = list(result["rules"].keys())[0]
+    except Exception as e:
+        return False
+    assert "policy_id" in result["rules"][rule_id]
+    assert policy_id == result["rules"][rule_id]["policy_id"]
+    assert "meta_rule_id" in result["rules"][rule_id]
+    assert meta_rule_id == result["rules"][rule_id]["meta_rule_id"]
+    assert rule == result["rules"][rule_id]["rule"]
+    return rule_id
+
+
+def check_rule(policy_id, meta_rule_id, rule_id, rule):
+    req = requests.get(URL.format("/policies/{}/rules".format(policy_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "rules" in result
+    assert "policy_id" in result["rules"]
+    assert policy_id == result["rules"]["policy_id"]
+    for item in result["rules"]["rules"]:
+        assert "meta_rule_id" in item
+        if meta_rule_id == item["meta_rule_id"]:
+            if rule_id == item["id"]:
+                assert rule == item["rule"]
+
+
+def delete_rule(policy_id, rule_id):
+    req = requests.delete(URL.format("/policies/{}/rules/{}".format(policy_id, rule_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "result" in result
+    assert result["result"]
+
+    req = requests.get(URL.format("/policies/{}/rules".format(policy_id)))
+    assert req.status_code == 200
+    result = req.json()
+    assert "rules" in result
+    assert "policy_id" in result["rules"]
+    assert policy_id == result["rules"]["policy_id"]
+    found_rule = False
+    for item in result["rules"]["rules"]:
+        if rule_id == item["id"]:
+            found_rule = True
+    assert not found_rule