Merge "python_moonclient stable"

16 files changed, 416 insertions, 478 deletions

diff --git a/moonv4/python_moonclient/python_moonclient/authz.py b/moonv4/python_moonclient/python_moonclient/authz.py
new file mode 100644
index 00000000..0ee62930
--- /dev/null
+++ b/moonv4/python_moonclient/python_moonclient/authz.py
@@ -0,0 +1,178 @@
+import copy
+import logging
+import threading
+import requests
+import time
+import json
+import random
+from uuid import uuid4
+
+HOST_MANAGER = None
+PORT_MANAGER = None
+HOST_KEYSTONE = None
+PORT_KEYSTONE = None
+
+lock = threading.Lock()
+logger = logging.getLogger(__name__)
+
+
+def _construct_payload(creds, current_rule, enforcer, target):
+    # Convert instances of object() in target temporarily to
+    # empty dict to avoid circular reference detection
+    # errors in jsonutils.dumps().
+    temp_target = copy.deepcopy(target)
+    for key in target.keys():
+        element = target.get(key)
+        if type(element) is object:
+            temp_target[key] = {}
+    _data = _json = None
+    if enforcer:
+        _data = {'rule': json.dumps(current_rule),
+                 'target': json.dumps(temp_target),
+                 'credentials': json.dumps(creds)}
+    else:
+        _json = {'rule': current_rule,
+                 'target': temp_target,
+                 'credentials': creds}
+    return _data, _json
+
+
+def _send(url, data=None, stress_test=False):
+    current_request = dict()
+    current_request['url'] = url
+    try:
+        if stress_test:
+            current_request['start'] = time.time()
+            # with lock:
+            res = requests.get(url)
+            current_request['end'] = time.time()
+            current_request['delta'] = current_request["end"] - current_request["start"]
+        else:
+            with lock:
+                current_request['start'] = time.time()
+                if data:
+                    data, _ = _construct_payload(data['credentials'], data['rule'], True, data['target'])
+                    res = requests.post(url, json=data,
+                                        headers={'content-type': "application/x-www-form-urlencode"}
+                                        )
+                else:
+                    res = requests.get(url)
+                current_request['end'] = time.time()
+                current_request['delta'] = current_request["end"] - current_request["start"]
+    except requests.exceptions.ConnectionError:
+        logger.warning("Unable to connect to server")
+        return {}
+    if not stress_test:
+        try:
+            j = res.json()
+            if res.status_code == 200:
+                logger.warning("\033[1m{}\033[m \033[32mGrant\033[m".format(url))
+            elif res.status_code == 401:
+                logger.warning("\033[1m{}\033[m \033[31mDeny\033[m".format(url))
+            else:
+                logger.error("\033[1m{}\033[m {} {}".format(url, res.status_code, res.text))
+        except Exception as e:
+            if res.text == "True":
+                logger.warning("\033[1m{}\033[m \033[32mGrant\033[m".format(url))
+            elif res.text == "False":
+                logger.warning("\033[1m{}\033[m \033[31mDeny\033[m".format(url))
+            else:
+                logger.error("\033[1m{}\033[m {} {}".format(url, res.status_code, res.text))
+                logger.exception(e)
+                logger.error(res.text)
+        else:
+            if j.get("result"):
+                # logger.warning("{} \033[32m{}\033[m".format(url, j.get("result")))
+                logger.debug("{}".format(j.get("error", "")))
+                current_request['result'] = "Grant"
+            else:
+                # logger.warning("{} \033[31m{}\033[m".format(url, "Deny"))
+                logger.debug("{}".format(j))
+                current_request['result'] = "Deny"
+    return current_request
+
+
+class AsyncGet(threading.Thread):
+
+    def __init__(self, url, semaphore=None, **kwargs):
+        threading.Thread.__init__(self)
+        self.url = url
+        self.kwargs = kwargs
+        self.sema = semaphore
+        self.result = dict()
+        self.uuid = uuid4().hex
+        self.index = kwargs.get("index", 0)
+
+    def run(self):
+        self.result = _send(self.url,
+                            data=self.kwargs.get("data"),
+                            stress_test=self.kwargs.get("stress_test", False))
+        self.result['index'] = self.index
+
+
+def send_requests(scenario, authz_host, authz_port, keystone_project_id, request_second=1, limit=500,
+                  dry_run=None, stress_test=False, destination="wrapper"):
+    backgrounds = []
+    time_data = list()
+    start_timing = time.time()
+    request_cpt = 0
+    SUBJECTS = tuple(scenario.subjects.keys())
+    OBJECTS = tuple(scenario.objects.keys())
+    ACTIONS = tuple(scenario.actions.keys())
+    while request_cpt < limit:
+        rule = (random.choice(SUBJECTS), random.choice(OBJECTS), random.choice(ACTIONS))
+        if destination.lower() == "wrapper":
+            url = "http://{}:{}/authz".format(authz_host, authz_port)
+            data = {
+                'target': {
+                    "user_id": random.choice(SUBJECTS),
+                    "target": {
+                        "name": random.choice(OBJECTS)
+                    },
+                    "project_id": keystone_project_id
+                },
+                'credentials': None,
+                'rule': random.choice(ACTIONS)
+            }
+        else:
+            url = "http://{}:{}/authz/{}/{}".format(authz_host, authz_port, keystone_project_id, "/".join(rule))
+            data = None
+        if dry_run:
+            logger.info(url)
+            continue
+        request_cpt += 1
+        if stress_test:
+            time_data.append(copy.deepcopy(_send(url, stress_test=stress_test)))
+        else:
+            background = AsyncGet(url, stress_test=stress_test, data=data,
+                                  index=request_cpt)
+            backgrounds.append(background)
+            background.start()
+        if request_second > 0:
+            if request_cpt % request_second == 0:
+                if time.time()-start_timing < 1:
+                    while True:
+                        if time.time()-start_timing > 1:
+                            break
+                start_timing = time.time()
+    if not stress_test:
+        for background in backgrounds:
+            background.join()
+            if background.result:
+                time_data.append(copy.deepcopy(background.result))
+    return time_data
+
+
+def save_data(filename, time_data):
+    json.dump(time_data, open(filename, "w"))
+
+
+def get_delta(time_data):
+    time_delta = list()
+    time_delta_sum1 = 0
+    for item in time_data:
+        time_delta.append(item['delta'])
+        time_delta_sum1 += item['delta']
+    time_delta_average1 = time_delta_sum1 / len(time_data)
+    return time_delta, time_delta_average1
+
diff --git a/moonv4/tests/utils/models.py b/moonv4/python_moonclient/python_moonclient/models.py
index 61fa6179..069c673b 100644
--- a/moonv4/tests/utils/models.py
+++ b/moonv4/python_moonclient/python_moonclient/models.py
@@ -1,6 +1,10 @@
+import logging
 import requests
 import copy
-import utils.config
+from . import config
+
+logger = logging.getLogger("moonclient.models")
+
 
 URL = None
 HEADERS = None
@@ -25,7 +29,7 @@ meta_rule_template = {
 
 
 def init(consul_host, consul_port):
-    conf_data = utils.config.get_config_data(consul_host, consul_port)
+    conf_data = config.get_config_data(consul_host, consul_port)
     global URL, HEADERS
     URL = "http://{}:{}".format(
         conf_data['manager_host'],
@@ -273,3 +277,43 @@ def add_meta_rule_to_model(model_id, meta_rule_id):
             assert result["result"]
         assert "meta_rules" in result['models'][model_id]
         assert meta_rule_list == result['models'][model_id]["meta_rules"]
+
+
+def create_model(scenario, model_id=None):
+    logger.info("Creating model {}".format(scenario.model_name))
+    if not model_id:
+        logger.info("Add model")
+        model_id = add_model(name=scenario.model_name)
+    logger.info("Add subject categories")
+    for cat in scenario.subject_categories:
+        scenario.subject_categories[cat] = add_subject_category(name=cat)
+    logger.info("Add object categories")
+    for cat in scenario.object_categories:
+        scenario.object_categories[cat] = add_object_category(name=cat)
+    logger.info("Add action categories")
+    for cat in scenario.action_categories:
+        scenario.action_categories[cat] = add_action_category(name=cat)
+    sub_cat = []
+    ob_cat = []
+    act_cat = []
+    meta_rule_list = []
+    for item_name, item_value in scenario.meta_rule.items():
+        for item in item_value["value"]:
+            if item in scenario.subject_categories:
+                sub_cat.append(scenario.subject_categories[item])
+            elif item in scenario.object_categories:
+                ob_cat.append(scenario.object_categories[item])
+            elif item in scenario.action_categories:
+                act_cat.append(scenario.action_categories[item])
+        meta_rules = check_meta_rule(meta_rule_id=None)
+        for _meta_rule_id, _meta_rule_value in meta_rules['meta_rules'].items():
+            if _meta_rule_value['name'] == item_name:
+                meta_rule_id = _meta_rule_id
+                break
+        else:
+            logger.info("Add meta rule")
+            meta_rule_id = add_meta_rule(item_name, sub_cat, ob_cat, act_cat)
+        item_value["id"] = meta_rule_id
+        if meta_rule_id not in meta_rule_list:
+            meta_rule_list.append(meta_rule_id)
+    return model_id, meta_rule_list
diff --git a/moonv4/tests/utils/parse.py b/moonv4/python_moonclient/python_moonclient/parse.py
index 34a4a996..34a4a996 100644
--- a/moonv4/tests/utils/parse.py
+++ b/moonv4/python_moonclient/python_moonclient/parse.py
diff --git a/moonv4/tests/utils/pdp.py b/moonv4/python_moonclient/python_moonclient/pdp.py
index 50998507..a7c75a61 100644
--- a/moonv4/tests/utils/pdp.py
+++ b/moonv4/python_moonclient/python_moonclient/pdp.py
@@ -1,6 +1,7 @@
+import sys
 import logging
 import requests
-import utils.config
+from python_moonclient import config
 
 logger = logging.getLogger("moonforming.utils.policies")
 URL = None
@@ -22,7 +23,7 @@ pdp_template = {
 
 
 def init(consul_host, consul_port):
-    conf_data = utils.config.get_config_data(consul_host, consul_port)
+    conf_data = config.get_config_data(consul_host, consul_port)
     global URL, HEADER, KEYSTONE_USER, KEYSTONE_PASSWORD, KEYSTONE_PROJECT, KEYSTONE_SERVER
     URL = "http://{}:{}".format(
         conf_data['manager_host'],
@@ -85,6 +86,22 @@ def get_keystone_projects():
     return req.json()
 
 
+def get_keystone_id(pdp_name):
+    keystone_project_id = None
+    for pdp_key, pdp_value in check_pdp()["pdps"].items():
+        if pdp_name:
+            if pdp_name != pdp_value["name"]:
+                continue
+        if pdp_value['security_pipeline'] and pdp_value["keystone_project_id"]:
+            logger.debug("Found pdp with keystone_project_id={}".format(pdp_value["keystone_project_id"]))
+            keystone_project_id = pdp_value["keystone_project_id"]
+
+    if not keystone_project_id:
+        logger.error("Cannot find PDP with keystone project ID")
+        sys.exit(1)
+    return keystone_project_id
+
+
 def check_pdp(pdp_id=None, keystone_project_id=None, moon_url=None):
     _URL = URL
     if moon_url:
@@ -173,3 +190,22 @@ def delete_pdp(pdp_id):
     assert type(result) is dict
     assert "result" in result
     assert result["result"]
+
+
+def create_pdp(scenario, policy_id=None, project_id=None):
+    logger.info("Creating PDP {}".format(scenario.pdp_name))
+    projects = get_keystone_projects()
+    if not project_id:
+        for _project in projects['projects']:
+            if _project['name'] == "admin":
+                project_id = _project['id']
+    assert project_id
+    pdps = check_pdp()["pdps"]
+    for pdp_id, pdp_value in pdps.items():
+        if scenario.pdp_name == pdp_value["name"]:
+            update_pdp(pdp_id, policy_id=policy_id)
+            logger.debug("Found existing PDP named {} (will add policy {})".format(scenario.pdp_name, policy_id))
+            return pdp_id
+    _pdp_id = add_pdp(name=scenario.pdp_name, policy_id=policy_id)
+    map_to_keystone(pdp_id=_pdp_id, keystone_project_id=project_id)
+    return _pdp_id
\ No newline at end of file
diff --git a/moonv4/tests/utils/policies.py b/moonv4/python_moonclient/python_moonclient/policies.py
index fd4d238f..80210811 100644
--- a/moonv4/tests/utils/policies.py
+++ b/moonv4/python_moonclient/python_moonclient/policies.py
@@ -1,6 +1,8 @@
 import logging
 import requests
-import utils.config
+from . import config, models
+
+logger = logging.getLogger("moonclient.models")
 
 URL = None
 HEADERS = None
@@ -54,7 +56,7 @@ subject_assignment_template = {
 
 
 def init(consul_host, consul_port):
-    conf_data = utils.config.get_config_data(consul_host, consul_port)
+    conf_data = config.get_config_data(consul_host, consul_port)
     global URL, HEADERS
     URL = "http://{}:{}".format(
         conf_data['manager_host'],
@@ -640,3 +642,122 @@ def delete_rule(policy_id, rule_id):
         if rule_id == item["id"]:
             found_rule = True
     assert not found_rule
+
+
+def create_policy(scenario, model_id, meta_rule_list):
+    logger.info("Creating policy {}".format(scenario.policy_name))
+    _policies = check_policy()
+    for _policy_id, _policy_value in _policies["policies"].items():
+        if _policy_value['name'] == scenario.policy_name:
+            policy_id = _policy_id
+            break
+    else:
+        policy_id = add_policy(name=scenario.policy_name, genre=scenario.policy_genre)
+
+    update_policy(policy_id, model_id)
+
+    for meta_rule_id in meta_rule_list:
+        logger.debug("add_meta_rule_to_model {} {}".format(model_id, meta_rule_id))
+        models.add_meta_rule_to_model(model_id, meta_rule_id)
+
+    logger.info("Add subject data")
+    for subject_cat_name in scenario.subject_data:
+        for subject_data_name in scenario.subject_data[subject_cat_name]:
+            data_id = scenario.subject_data[subject_cat_name][subject_data_name] = add_subject_data(
+                policy_id=policy_id,
+                category_id=scenario.subject_categories[subject_cat_name], name=subject_data_name)
+            scenario.subject_data[subject_cat_name][subject_data_name] = data_id
+    logger.info("Add object data")
+    for object_cat_name in scenario.object_data:
+        for object_data_name in scenario.object_data[object_cat_name]:
+            data_id = scenario.object_data[object_cat_name][object_data_name] = add_object_data(
+                policy_id=policy_id,
+                category_id=scenario.object_categories[object_cat_name], name=object_data_name)
+            scenario.object_data[object_cat_name][object_data_name] = data_id
+    logger.info("Add action data")
+    for action_cat_name in scenario.action_data:
+        for action_data_name in scenario.action_data[action_cat_name]:
+            data_id = scenario.action_data[action_cat_name][action_data_name] = add_action_data(
+                policy_id=policy_id,
+                category_id=scenario.action_categories[action_cat_name], name=action_data_name)
+            scenario.action_data[action_cat_name][action_data_name] = data_id
+
+    logger.info("Add subjects")
+    for name in scenario.subjects:
+        scenario.subjects[name] = add_subject(policy_id, name=name)
+    logger.info("Add objects")
+    for name in scenario.objects:
+        scenario.objects[name] = add_object(policy_id, name=name)
+    logger.info("Add actions")
+    for name in scenario.actions:
+        scenario.actions[name] = add_action(policy_id, name=name)
+
+    logger.info("Add subject assignments")
+    for subject_name in scenario.subject_assignments:
+        if type(scenario.subject_assignments[subject_name]) in (list, tuple):
+            for items in scenario.subject_assignments[subject_name]:
+                for subject_category_name in items:
+                    subject_id = scenario.subjects[subject_name]
+                    subject_cat_id = scenario.subject_categories[subject_category_name]
+                    for data in scenario.subject_assignments[subject_name]:
+                        subject_data_id = scenario.subject_data[subject_category_name][data[subject_category_name]]
+                        add_subject_assignments(policy_id, subject_id, subject_cat_id, subject_data_id)
+        else:
+            for subject_category_name in scenario.subject_assignments[subject_name]:
+                subject_id = scenario.subjects[subject_name]
+                subject_cat_id = scenario.subject_categories[subject_category_name]
+                subject_data_id = scenario.subject_data[subject_category_name][scenario.subject_assignments[subject_name][subject_category_name]]
+                add_subject_assignments(policy_id, subject_id, subject_cat_id, subject_data_id)
+
+    logger.info("Add object assignments")
+    for object_name in scenario.object_assignments:
+        if type(scenario.object_assignments[object_name]) in (list, tuple):
+            for items in scenario.object_assignments[object_name]:
+                for object_category_name in items:
+                    object_id = scenario.objects[object_name]
+                    object_cat_id = scenario.object_categories[object_category_name]
+                    for data in scenario.object_assignments[object_name]:
+                        object_data_id = scenario.object_data[object_category_name][data[object_category_name]]
+                        add_object_assignments(policy_id, object_id, object_cat_id, object_data_id)
+        else:
+            for object_category_name in scenario.object_assignments[object_name]:
+                object_id = scenario.objects[object_name]
+                object_cat_id = scenario.object_categories[object_category_name]
+                object_data_id = scenario.object_data[object_category_name][scenario.object_assignments[object_name][object_category_name]]
+                add_object_assignments(policy_id, object_id, object_cat_id, object_data_id)
+
+    logger.info("Add action assignments")
+    for action_name in scenario.action_assignments:
+        if type(scenario.action_assignments[action_name]) in (list, tuple):
+            for items in scenario.action_assignments[action_name]:
+                for action_category_name in items:
+                    action_id = scenario.actions[action_name]
+                    action_cat_id = scenario.action_categories[action_category_name]
+                    for data in scenario.action_assignments[action_name]:
+                        action_data_id = scenario.action_data[action_category_name][data[action_category_name]]
+                        add_action_assignments(policy_id, action_id, action_cat_id, action_data_id)
+        else:
+            for action_category_name in scenario.action_assignments[action_name]:
+                action_id = scenario.actions[action_name]
+                action_cat_id = scenario.action_categories[action_category_name]
+                action_data_id = scenario.action_data[action_category_name][scenario.action_assignments[action_name][action_category_name]]
+                add_action_assignments(policy_id, action_id, action_cat_id, action_data_id)
+
+    logger.info("Add rules")
+    for meta_rule_name in scenario.rules:
+        meta_rule_value = scenario.meta_rule[meta_rule_name]
+        for rule in scenario.rules[meta_rule_name]:
+            data_list = []
+            _meta_rule = list(meta_rule_value["value"])
+            for data_name in rule["rule"]:
+                category_name = _meta_rule.pop(0)
+                if category_name in scenario.subject_categories:
+                    data_list.append(scenario.subject_data[category_name][data_name])
+                elif category_name in scenario.object_categories:
+                    data_list.append(scenario.object_data[category_name][data_name])
+                elif category_name in scenario.action_categories:
+                    data_list.append(scenario.action_data[category_name][data_name])
+            instructions = rule["instructions"]
+            add_rule(policy_id, meta_rule_value["id"], data_list, instructions)
+    return policy_id
+
diff --git a/moonv4/python_moonclient/tests/unit_python/conftest.py b/moonv4/python_moonclient/tests/unit_python/conftest.py
index d26df946..e98f48c5 100644
--- a/moonv4/python_moonclient/tests/unit_python/conftest.py
+++ b/moonv4/python_moonclient/tests/unit_python/conftest.py
@@ -1,6 +1,6 @@
 import pytest
 import requests_mock
-import mock_config
+from . import mock_config
 
 
 @pytest.fixture(autouse=True)
diff --git a/moonv4/python_moonclient/tests/unit_python/mock_config.py b/moonv4/python_moonclient/tests/unit_python/mock_config.py
index a3084485..6d6c8249 100644
--- a/moonv4/python_moonclient/tests/unit_python/mock_config.py
+++ b/moonv4/python_moonclient/tests/unit_python/mock_config.py
@@ -1,4 +1,4 @@
-import utilities
+from . import utilities
 
 
 components_manager_mock = {
diff --git a/moonv4/python_moonclient/tests/unit_python/test_config.py b/moonv4/python_moonclient/tests/unit_python/test_config.py
index 21b5f630..ebdfacf0 100644
--- a/moonv4/python_moonclient/tests/unit_python/test_config.py
+++ b/moonv4/python_moonclient/tests/unit_python/test_config.py
@@ -1,5 +1,5 @@
 import pytest
-import utilities
+from . import utilities
 
 
 def test_authz_request():
diff --git a/moonv4/tests/test_models.py b/moonv4/python_moonclient/tests/unit_python/test_models.py
index 0da40ce5..f708c6e4 100644
--- a/moonv4/tests/test_models.py
+++ b/moonv4/python_moonclient/tests/unit_python/test_models.py
@@ -1,4 +1,4 @@
-from utils.models import *
+from python_moonclient.models import *
 
 
 def test_models():
diff --git a/moonv4/tests/test_pdp.py b/moonv4/python_moonclient/tests/unit_python/test_pdp.py
index 6cd5365b..8d9a3ac3 100644
--- a/moonv4/tests/test_pdp.py
+++ b/moonv4/python_moonclient/tests/unit_python/test_pdp.py
@@ -1,4 +1,4 @@
-from utils.pdp import *
+from python_moonclient.pdp import *
 
 
 def test_pdp():
diff --git a/moonv4/tests/test_policies.py b/moonv4/python_moonclient/tests/unit_python/test_policies.py
index 8f26d72d..386c37af 100644
--- a/moonv4/tests/test_policies.py
+++ b/moonv4/python_moonclient/tests/unit_python/test_policies.py
@@ -1,5 +1,5 @@
-from utils.policies import *
-from utils.models import *
+from python_moonclient.policies import *
+from python_moonclient.models import *
 
 
 def test_policies():
diff --git a/moonv4/tests/get_keystone_projects.py b/moonv4/tests/get_keystone_projects.py
index 7b37b0e7..9b5d87cd 100644
--- a/moonv4/tests/get_keystone_projects.py
+++ b/moonv4/tests/get_keystone_projects.py
@@ -1,8 +1,5 @@
-from utils import pdp
-from utils import parse
-from utils import models
-from utils import policies
-from utils import pdp
+from python_moonclient import parse, models, policies, pdp
+
 
 if __name__ == "__main__":
     args = parse.parse()
diff --git a/moonv4/tests/populate_default_values.py b/moonv4/tests/populate_default_values.py
index 28795526..d5a5769b 100644
--- a/moonv4/tests/populate_default_values.py
+++ b/moonv4/tests/populate_default_values.py
@@ -1,194 +1,11 @@
 import logging
 from importlib.machinery import SourceFileLoader
-from utils import parse
-from utils import models
-from utils import policies
-from utils import pdp
+from python_moonclient import parse, models, policies, pdp
 
-logger = None
-
-
-def create_model(model_id=None):
-    if args.verbose:
-        logger.info("Creating model {}".format(scenario.model_name))
-    if not model_id:
-        logger.info("Add model")
-        model_id = models.add_model(name=scenario.model_name)
-    logger.info("Add subject categories")
-    for cat in scenario.subject_categories:
-        scenario.subject_categories[cat] = models.add_subject_category(name=cat)
-    logger.info("Add object categories")
-    for cat in scenario.object_categories:
-        scenario.object_categories[cat] = models.add_object_category(name=cat)
-    logger.info("Add action categories")
-    for cat in scenario.action_categories:
-        scenario.action_categories[cat] = models.add_action_category(name=cat)
-    sub_cat = []
-    ob_cat = []
-    act_cat = []
-    meta_rule_list = []
-    for item_name, item_value in scenario.meta_rule.items():
-        for item in item_value["value"]:
-            if item in scenario.subject_categories:
-                sub_cat.append(scenario.subject_categories[item])
-            elif item in scenario.object_categories:
-                ob_cat.append(scenario.object_categories[item])
-            elif item in scenario.action_categories:
-                act_cat.append(scenario.action_categories[item])
-        meta_rules = models.check_meta_rule(meta_rule_id=None)
-        for _meta_rule_id, _meta_rule_value in meta_rules['meta_rules'].items():
-            if _meta_rule_value['name'] == item_name:
-                meta_rule_id = _meta_rule_id
-                break
-        else:
-            logger.info("Add meta rule")
-            meta_rule_id = models.add_meta_rule(item_name, sub_cat, ob_cat, act_cat)
-        item_value["id"] = meta_rule_id
-        if meta_rule_id not in meta_rule_list:
-            meta_rule_list.append(meta_rule_id)
-    return model_id, meta_rule_list
-
-
-def create_policy(model_id, meta_rule_list):
-    if args.verbose:
-        logger.info("Creating policy {}".format(scenario.policy_name))
-    _policies = policies.check_policy()
-    for _policy_id, _policy_value in _policies["policies"].items():
-        if _policy_value['name'] == scenario.policy_name:
-            policy_id = _policy_id
-            break
-    else:
-        policy_id = policies.add_policy(name=scenario.policy_name, genre=scenario.policy_genre)
-
-    policies.update_policy(policy_id, model_id)
-
-    for meta_rule_id in meta_rule_list:
-        logger.debug("add_meta_rule_to_model {} {}".format(model_id, meta_rule_id))
-        models.add_meta_rule_to_model(model_id, meta_rule_id)
-
-    logger.info("Add subject data")
-    for subject_cat_name in scenario.subject_data:
-        for subject_data_name in scenario.subject_data[subject_cat_name]:
-            data_id = scenario.subject_data[subject_cat_name][subject_data_name] = policies.add_subject_data(
-                policy_id=policy_id,
-                category_id=scenario.subject_categories[subject_cat_name], name=subject_data_name)
-            scenario.subject_data[subject_cat_name][subject_data_name] = data_id
-    logger.info("Add object data")
-    for object_cat_name in scenario.object_data:
-        for object_data_name in scenario.object_data[object_cat_name]:
-            data_id = scenario.object_data[object_cat_name][object_data_name] = policies.add_object_data(
-                policy_id=policy_id,
-                category_id=scenario.object_categories[object_cat_name], name=object_data_name)
-            scenario.object_data[object_cat_name][object_data_name] = data_id
-    logger.info("Add action data")
-    for action_cat_name in scenario.action_data:
-        for action_data_name in scenario.action_data[action_cat_name]:
-            data_id = scenario.action_data[action_cat_name][action_data_name] = policies.add_action_data(
-                policy_id=policy_id,
-                category_id=scenario.action_categories[action_cat_name], name=action_data_name)
-            scenario.action_data[action_cat_name][action_data_name] = data_id
-
-    logger.info("Add subjects")
-    for name in scenario.subjects:
-        scenario.subjects[name] = policies.add_subject(policy_id, name=name)
-    logger.info("Add objects")
-    for name in scenario.objects:
-        scenario.objects[name] = policies.add_object(policy_id, name=name)
-    logger.info("Add actions")
-    for name in scenario.actions:
-        scenario.actions[name] = policies.add_action(policy_id, name=name)
-
-    logger.info("Add subject assignments")
-    for subject_name in scenario.subject_assignments:
-        if type(scenario.subject_assignments[subject_name]) in (list, tuple):
-            for items in scenario.subject_assignments[subject_name]:
-                for subject_category_name in items:
-                    subject_id = scenario.subjects[subject_name]
-                    subject_cat_id = scenario.subject_categories[subject_category_name]
-                    for data in scenario.subject_assignments[subject_name]:
-                        subject_data_id = scenario.subject_data[subject_category_name][data[subject_category_name]]
-                        policies.add_subject_assignments(policy_id, subject_id, subject_cat_id, subject_data_id)
-        else:
-            for subject_category_name in scenario.subject_assignments[subject_name]:
-                subject_id = scenario.subjects[subject_name]
-                subject_cat_id = scenario.subject_categories[subject_category_name]
-                subject_data_id = scenario.subject_data[subject_category_name][scenario.subject_assignments[subject_name][subject_category_name]]
-                policies.add_subject_assignments(policy_id, subject_id, subject_cat_id, subject_data_id)
-
-    logger.info("Add object assignments")
-    for object_name in scenario.object_assignments:
-        if type(scenario.object_assignments[object_name]) in (list, tuple):
-            for items in scenario.object_assignments[object_name]:
-                for object_category_name in items:
-                    object_id = scenario.objects[object_name]
-                    object_cat_id = scenario.object_categories[object_category_name]
-                    for data in scenario.object_assignments[object_name]:
-                        object_data_id = scenario.object_data[object_category_name][data[object_category_name]]
-                        policies.add_object_assignments(policy_id, object_id, object_cat_id, object_data_id)
-        else:
-            for object_category_name in scenario.object_assignments[object_name]:
-                object_id = scenario.objects[object_name]
-                object_cat_id = scenario.object_categories[object_category_name]
-                object_data_id = scenario.object_data[object_category_name][scenario.object_assignments[object_name][object_category_name]]
-                policies.add_object_assignments(policy_id, object_id, object_cat_id, object_data_id)
-
-    logger.info("Add action assignments")
-    for action_name in scenario.action_assignments:
-        if type(scenario.action_assignments[action_name]) in (list, tuple):
-            for items in scenario.action_assignments[action_name]:
-                for action_category_name in items:
-                    action_id = scenario.actions[action_name]
-                    action_cat_id = scenario.action_categories[action_category_name]
-                    for data in scenario.action_assignments[action_name]:
-                        action_data_id = scenario.action_data[action_category_name][data[action_category_name]]
-                        policies.add_action_assignments(policy_id, action_id, action_cat_id, action_data_id)
-        else:
-            for action_category_name in scenario.action_assignments[action_name]:
-                action_id = scenario.actions[action_name]
-                action_cat_id = scenario.action_categories[action_category_name]
-                action_data_id = scenario.action_data[action_category_name][scenario.action_assignments[action_name][action_category_name]]
-                policies.add_action_assignments(policy_id, action_id, action_cat_id, action_data_id)
-
-    logger.info("Add rules")
-    for meta_rule_name in scenario.rules:
-        meta_rule_value = scenario.meta_rule[meta_rule_name]
-        for rule in scenario.rules[meta_rule_name]:
-            data_list = []
-            _meta_rule = list(meta_rule_value["value"])
-            for data_name in rule["rule"]:
-                category_name = _meta_rule.pop(0)
-                if category_name in scenario.subject_categories:
-                    data_list.append(scenario.subject_data[category_name][data_name])
-                elif category_name in scenario.object_categories:
-                    data_list.append(scenario.object_data[category_name][data_name])
-                elif category_name in scenario.action_categories:
-                    data_list.append(scenario.action_data[category_name][data_name])
-            instructions = rule["instructions"]
-            policies.add_rule(policy_id, meta_rule_value["id"], data_list, instructions)
-    return policy_id
-
-
-def create_pdp(policy_id=None, project_id=None):
-    logger.info("Creating PDP {}".format(scenario.pdp_name))
-    projects = pdp.get_keystone_projects()
-    if not project_id:
-        for _project in projects['projects']:
-            if _project['name'] == "admin":
-                project_id = _project['id']
-    assert project_id
-    pdps = pdp.check_pdp()["pdps"]
-    for pdp_id, pdp_value in pdps.items():
-        if scenario.pdp_name == pdp_value["name"]:
-            pdp.update_pdp(pdp_id, policy_id=policy_id)
-            logger.debug("Found existing PDP named {} (will add policy {})".format(scenario.pdp_name, policy_id))
-            return pdp_id
-    _pdp_id = pdp.add_pdp(name=scenario.pdp_name, policy_id=policy_id)
-    pdp.map_to_keystone(pdp_id=_pdp_id, keystone_project_id=project_id)
-    return _pdp_id
+logger = logging.getLogger("moonforming")
 
 
 if __name__ == "__main__":
-    logger = logging.getLogger("moonforming")
     requests_log = logging.getLogger("requests.packages.urllib3")
     requests_log.setLevel(logging.WARNING)
     requests_log.propagate = True
@@ -212,9 +29,9 @@ if __name__ == "__main__":
         if _model_value['name'] == scenario.model_name:
             model_id = _model_id
             meta_rule_list = _model_value['meta_rules']
-            create_model(model_id)
+            models.create_model(scenario, model_id)
             break
     else:
-        model_id, meta_rule_list = create_model()
-    policy_id = create_policy(model_id, meta_rule_list)
-    pdp_id = create_pdp(policy_id=policy_id, project_id=project_id)
+        model_id, meta_rule_list = models.create_model(scenario)
+    policy_id = policies.create_policy(scenario, model_id, meta_rule_list)
+    pdp_id = pdp.create_pdp(scenario, policy_id=policy_id, project_id=project_id)
diff --git a/moonv4/tests/send_authz.py b/moonv4/tests/send_authz.py
index 5766a0ec..b4ed1d2f 100644
--- a/moonv4/tests/send_authz.py
+++ b/moonv4/tests/send_authz.py
@@ -1,233 +1,26 @@
-import sys
-import copy
-import logging
-import threading
 from importlib.machinery import SourceFileLoader
-import requests
-import time
-import json
-import random
-from uuid import uuid4
-from utils.pdp import check_pdp
-from utils.parse import parse
-import utils.config
+from python_moonclient import config, parse, models, policies, pdp, authz
 
 
-logger = None
-HOST_MANAGER = None
-PORT_MANAGER = None
-HOST_AUTHZ = None
-PORT_AUTHZ = None
-HOST_KEYSTONE = None
-PORT_KEYSTONE = None
-
-lock = threading.Lock()
-logger = logging.getLogger(__name__)
-
-
-def get_scenario(args):
-    m = SourceFileLoader("scenario", args.filename[0])
-    return m.load_module()
-
-
-def get_keystone_id(pdp_name):
-    global HOST_MANAGER, PORT_MANAGER
-    keystone_project_id = None
-    for pdp_key, pdp_value in check_pdp(moon_url="http://{}:{}".format(HOST_MANAGER, PORT_MANAGER))["pdps"].items():
-        if pdp_name:
-            if pdp_name != pdp_value["name"]:
-                continue
-        if pdp_value['security_pipeline'] and pdp_value["keystone_project_id"]:
-            logger.debug("Found pdp with keystone_project_id={}".format(pdp_value["keystone_project_id"]))
-            keystone_project_id = pdp_value["keystone_project_id"]
-
-    if not keystone_project_id:
-        logger.error("Cannot find PDP with keystone project ID")
-        sys.exit(1)
-    return keystone_project_id
-
-
-def _construct_payload(creds, current_rule, enforcer, target):
-    # Convert instances of object() in target temporarily to
-    # empty dict to avoid circular reference detection
-    # errors in jsonutils.dumps().
-    temp_target = copy.deepcopy(target)
-    for key in target.keys():
-        element = target.get(key)
-        if type(element) is object:
-            temp_target[key] = {}
-    _data = _json = None
-    if enforcer:
-        _data = {'rule': json.dumps(current_rule),
-                 'target': json.dumps(temp_target),
-                 'credentials': json.dumps(creds)}
-    else:
-        _json = {'rule': current_rule,
-                 'target': temp_target,
-                 'credentials': creds}
-    return _data, _json
-
-
-def _send(url, data=None, stress_test=False):
-    current_request = dict()
-    current_request['url'] = url
-    try:
-        if stress_test:
-            current_request['start'] = time.time()
-            # with lock:
-            res = requests.get(url)
-            current_request['end'] = time.time()
-            current_request['delta'] = current_request["end"] - current_request["start"]
-        else:
-            with lock:
-                current_request['start'] = time.time()
-                if data:
-                    data, _ = _construct_payload(data['credentials'], data['rule'], True, data['target'])
-                    res = requests.post(url, json=data,
-                                        headers={'content-type': "application/x-www-form-urlencode"}
-                                        )
-                else:
-                    res = requests.get(url)
-                current_request['end'] = time.time()
-                current_request['delta'] = current_request["end"] - current_request["start"]
-    except requests.exceptions.ConnectionError:
-        logger.warning("Unable to connect to server")
-        return {}
-    if not stress_test:
-        try:
-            j = res.json()
-            if res.status_code == 200:
-                logger.warning("\033[1m{}\033[m \033[32mGrant\033[m".format(url))
-            elif res.status_code == 401:
-                logger.warning("\033[1m{}\033[m \033[31mDeny\033[m".format(url))
-            else:
-                logger.error("\033[1m{}\033[m {} {}".format(url, res.status_code, res.text))
-        except Exception as e:
-            if res.text == "True":
-                logger.warning("\033[1m{}\033[m \033[32mGrant\033[m".format(url))
-            elif res.text == "False":
-                logger.warning("\033[1m{}\033[m \033[31mDeny\033[m".format(url))
-            else:
-                logger.error("\033[1m{}\033[m {} {}".format(url, res.status_code, res.text))
-                logger.exception(e)
-                logger.error(res.text)
-        else:
-            if j.get("result"):
-                # logger.warning("{} \033[32m{}\033[m".format(url, j.get("result")))
-                logger.debug("{}".format(j.get("error", "")))
-                current_request['result'] = "Grant"
-            else:
-                # logger.warning("{} \033[31m{}\033[m".format(url, "Deny"))
-                logger.debug("{}".format(j))
-                current_request['result'] = "Deny"
-    return current_request
-
-
-class AsyncGet(threading.Thread):
-
-    def __init__(self, url, semaphore=None, **kwargs):
-        threading.Thread.__init__(self)
-        self.url = url
-        self.kwargs = kwargs
-        self.sema = semaphore
-        self.result = dict()
-        self.uuid = uuid4().hex
-        self.index = kwargs.get("index", 0)
-
-    def run(self):
-        self.result = _send(self.url,
-                            data=self.kwargs.get("data"),
-                            stress_test=self.kwargs.get("stress_test", False))
-        self.result['index'] = self.index
-
-
-def send_requests(scenario, keystone_project_id, request_second=1, limit=500,
-                  dry_run=None, stress_test=False, destination="wrapper"):
-    global HOST_AUTHZ, PORT_AUTHZ
-    backgrounds = []
-    time_data = list()
-    start_timing = time.time()
-    request_cpt = 0
-    SUBJECTS = tuple(scenario.subjects.keys())
-    OBJECTS = tuple(scenario.objects.keys())
-    ACTIONS = tuple(scenario.actions.keys())
-    while request_cpt < limit:
-        rule = (random.choice(SUBJECTS), random.choice(OBJECTS), random.choice(ACTIONS))
-        if destination.lower() == "wrapper":
-            url = "http://{}:{}/authz".format(HOST_AUTHZ, PORT_AUTHZ)
-            data = {
-                'target': {
-                    "user_id": random.choice(SUBJECTS),
-                    "target": {
-                        "name": random.choice(OBJECTS)
-                    },
-                    "project_id": keystone_project_id
-                },
-                'credentials': None,
-                'rule': random.choice(ACTIONS)
-            }
-        else:
-            url = "http://{}:{}/authz/{}/{}".format(HOST_AUTHZ, PORT_AUTHZ, keystone_project_id, "/".join(rule))
-            data = None
-        if dry_run:
-            logger.info(url)
-            continue
-        request_cpt += 1
-        if stress_test:
-            time_data.append(copy.deepcopy(_send(url, stress_test=stress_test)))
-        else:
-            background = AsyncGet(url, stress_test=stress_test, data=data,
-                                  index=request_cpt)
-            backgrounds.append(background)
-            background.start()
-        if request_second > 0:
-            if request_cpt % request_second == 0:
-                if time.time()-start_timing < 1:
-                    while True:
-                        if time.time()-start_timing > 1:
-                            break
-                start_timing = time.time()
-    if not stress_test:
-        for background in backgrounds:
-            background.join()
-            if background.result:
-                time_data.append(copy.deepcopy(background.result))
-    return time_data
-
-
-def save_data(filename, time_data):
-    json.dump(time_data, open(filename, "w"))
-
-
-def get_delta(time_data):
-    time_delta = list()
-    time_delta_sum1 = 0
-    for item in time_data:
-        time_delta.append(item['delta'])
-        time_delta_sum1 += item['delta']
-    time_delta_average1 = time_delta_sum1 / len(time_data)
-    return time_delta, time_delta_average1
-
-
-def main():
-    global HOST_MANAGER, PORT_MANAGER, HOST_AUTHZ, PORT_AUTHZ
-
-    args = parse()
+if __name__ == "__main__":
+    args = parse.parse()
     consul_host = args.consul_host
     consul_port = args.consul_port
-    conf_data = utils.config.get_config_data(consul_host, consul_port)
 
-    HOST_MANAGER = conf_data['manager_host']
-    PORT_MANAGER = conf_data['manager_port']
-    HOST_AUTHZ = args.authz_host
-    PORT_AUTHZ = args.authz_port
-    # HOST_KEYSTONE = conf_data['keystone_host']
-    # PORT_KEYSTONE = conf_data['manager_host']
+    models.init(consul_host, consul_port)
+    policies.init(consul_host, consul_port)
+    pdp.init(consul_host, consul_port)
+
+    if args.filename:
+        print("Loading: {}".format(args.filename[0]))
+    m = SourceFileLoader("scenario", args.filename[0])
+    scenario = m.load_module()
 
-    scenario = get_scenario(args)
-    keystone_project_id = get_keystone_id(args.pdp)
-    time_data = send_requests(
+    keystone_project_id = pdp.get_keystone_id(args.pdp)
+    time_data = authz.send_requests(
         scenario,
+        args.authz_host,
+        args.authz_port,
         keystone_project_id,
         request_second=args.request_second,
         limit=args.limit,
@@ -236,8 +29,4 @@ def main():
         destination=args.destination
     )
     if not args.dry_run:
-        save_data(args.write, time_data)
-
-
-if __name__ == "__main__":
-    main()
+        authz.save_data(args.write, time_data)
diff --git a/moonv4/tests/utils/__init__.py b/moonv4/tests/utils/__init__.py
deleted file mode 100644
index e69de29b..00000000
--- a/moonv4/tests/utils/__init__.py
+++ /dev/null
diff --git a/moonv4/tests/utils/config.py b/moonv4/tests/utils/config.py
deleted file mode 100644
index d6317820..00000000
--- a/moonv4/tests/utils/config.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import base64
-import json
-import requests
-
-
-def get_configuration(consul_host, consul_port, key):
-    url = "http://{}:{}/v1/kv/{}".format(consul_host, consul_port, key)
-    req = requests.get(url)
-    if req.status_code != 200:
-        raise Exception("xxx")
-    data = req.json()
-    if len(data) == 1:
-        data = data[0]
-        return {data["Key"]: json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
-    else:
-        return [
-            {item["Key"]: json.loads(base64.b64decode(item["Value"]).decode("utf-8"))}
-            for item in data
-        ]
-
-
-def get_config_data(consul_host, consul_port):
-    conf_data = dict()
-    conf_data['manager_host'] = get_configuration(consul_host, consul_port,
-                                                  'components/manager')['components/manager']['external']['hostname']
-    conf_data['manager_port'] = get_configuration(consul_host, consul_port,
-                                                  'components/manager')['components/manager']['external']['port']
-    # conf_data['authz_host'] = get_configuration(consul_host, consul_port,
-    #                                             'components/interface')['components/interface']['external']['hostname']
-    # conf_data['authz_port'] = get_configuration(consul_host, consul_port,
-    #                                             'components/interface')['components/interface']['external']['port']
-    conf_data['keystone_host'] = get_configuration(consul_host, consul_port,
-                                                   'openstack/keystone')['openstack/keystone']['external']['url']
-    # conf_data['keystone_port'] = '5000'
-    conf_data['keystone_user'] = get_configuration(consul_host, consul_port,
-                                                   'openstack/keystone')['openstack/keystone']['user']
-    conf_data['keystone_password'] = get_configuration(consul_host, consul_port,
-                                                       'openstack/keystone')['openstack/keystone']['password']
-    conf_data['keystone_project'] = get_configuration(consul_host, consul_port,
-                                                      'openstack/keystone')['openstack/keystone']['project']
-    return conf_data
-
-# get_conf_data('88.88.88.2', '30005')
-# get_conf_data('127.0.0.1', 8082)