diff options
51 files changed, 1340 insertions, 248 deletions
diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..42756ece --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,23 @@ +properties([ + pipelineTriggers([cron('H * * * *')]) +]) +node { + checkout scm + def packages = ["python_moonutilities","python_moondb","python_moonclient"] + def subtests = [:] + for (x in packages) { + def pkg = x + subtests[pkg] = { + withDockerContainer('wukongsun/moon_python_unit_test') { + stage("Install prerequisites for package ${pkg}") { + sh("pip install pytest requests_mock requests --upgrade") + sh("cd ${pkg} && pip install -r tests/unit_python/requirements.txt && pip install .") + } + stage("Unit test for package ${pkg}") { + sh "cd ${pkg}/tests/unit_python && pytest ." + } + } + } + } + parallel subtests +}
\ No newline at end of file @@ -43,6 +43,32 @@ curl http://$MOON_HOST:30001/pdp curl http://$MOON_HOST:30001/policies ``` +The Moon platform is fully installed and configured when you have no error with the `moon_get_keystone_projects`: +```bash +sudo pip install python_moonclient --upgrade +moon_get_keystone_projects +``` + +### moon_wrapper +The moon_wrapper component is used to connect OpenStack to the Moon platform. +You need to load one wrapper before connecting OpenStack to Moon. +First of all, get the names of all available slaves: +```bash +moon_get_slaves +``` +Select the slave you want to configure: +```bash +moon_set_slave <name_of_the_slave> +``` +If you don't put a name here, by default, the script will use `kubernetes-admin@kubernetes` +which is the master. + +If you need to unload the slave, use the following command: +```bash +moon_delete_slave <name_of_the_slave> +``` +If you don't put a name here, by default, the script will use `kubernetes-admin@kubernetes`. + ## Tests - [Python Unit Test](tests/python_unit/README.md) diff --git a/docs/TODO.md b/docs/TODO.md index afdadf3c..caca158a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -21,6 +21,8 @@ Bugs to fix: - When adding user or VM in GUI, there is a bug in the backend (manager ?) - GUI: in the "Projects" tab, move the "Map" link in the "Action" button - GUI: move tabs in this order : "Models, Policy, PDP, Projects" +- Fixing Error Handling at module "Interface" & "Wrapper" according to this link 'http://flask-restful.readthedocs.io/en/0.3.5/extending.html#custom-error-handlers' +- in case an error occurred, it would be better throwing an exception rather than sending result with error code (as the exception was already thrown in some cases ) [ Interface, Wrapper] Other actions: diff --git a/moon_forming/Dockerfile b/moon_forming/Dockerfile index ca0eba76..74616c89 100644 --- a/moon_forming/Dockerfile +++ b/moon_forming/Dockerfile @@ -1,11 +1,11 @@ FROM python:3 WORKDIR /usr/src/app -RUN pip install --no-cache-dir --upgrade requests pyyaml python_moonutilities python_moondb python_moonclient +RUN pip install --no-cache-dir --upgrade requests pytest pyyaml python_moonutilities python_moondb python_moonclient -ENV POPULATE_ARGS "-v" +ENV COMMAND "config" ADD . /root WORKDIR /root -CMD sh /root/run.sh ${POPULATE_ARGS}
\ No newline at end of file +CMD /bin/bash /root/switch.sh ${COMMAND} diff --git a/moon_forming/README.md b/moon_forming/README.md index cc08f676..9b755d96 100644 --- a/moon_forming/README.md +++ b/moon_forming/README.md @@ -39,6 +39,9 @@ kubectl delete -f $MOON_HOME/tools/moon_kubernetes/templates/moon_forming.yaml kubectl create -f $MOON_HOME/tools/moon_kubernetes/templates/moon_forming.yaml ``` +## Functional tests - - +```bash +cd $MOON_HOME/moon_manager +bash ../tests/functional/run_tests_for_component.sh +``` diff --git a/moon_forming/run.sh b/moon_forming/config_moon.sh index d731cb17..0a55898f 100644 --- a/moon_forming/run.sh +++ b/moon_forming/config_moon.sh @@ -37,7 +37,3 @@ while ! python -c "import requests; req = requests.get('http://manager:8082')" 2 done echo "." echo "Manager (http://manager:8082) is up." - -#for i in /data/*.py ; do -# moon_populate_values $populate_args --consul-host=consul --consul-port=8500 $i -#done diff --git a/moon_forming/switch.sh b/moon_forming/switch.sh new file mode 100644 index 00000000..adb1ebe9 --- /dev/null +++ b/moon_forming/switch.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CMD=$1 + +echo "COMMAND IS ${CMD}" + +if [ "${CMD}" = "functest" ]; then + echo "FUNCTIONAL TESTS" + ls -l /data + ls -l /data/tests + sh /data/tests/functional_pod/run_functional_tests.sh +#elif [ $CMD == "unittest" ]; then +# sh /data/tests/functional_pod/run_functional_tests.sh +else + echo "CONFIGURATION" + bash config_moon.sh +fi + +echo "<END OF JOB>"
\ No newline at end of file diff --git a/moon_gui/static/version.json b/moon_gui/static/version.json index ec74a2db..4f228d2b 100755 --- a/moon_gui/static/version.json +++ b/moon_gui/static/version.json @@ -1,3 +1,3 @@ { - "version": "1.0.0" -}
\ No newline at end of file + "version": "1.1.0" +} diff --git a/moon_interface/moon_interface/authz_requests.py b/moon_interface/moon_interface/authz_requests.py index 12c190c7..2ef0e0a1 100644 --- a/moon_interface/moon_interface/authz_requests.py +++ b/moon_interface/moon_interface/authz_requests.py @@ -33,7 +33,7 @@ class AuthzRequest: raise exceptions.KeystoneProjectError("Unknown Project ID {}".format(ctx['project_id'])) self.container_chaining = CACHE.container_chaining[ctx['project_id']] - if len(self.container_chaining) == 0 or not all(k in self.container_data for k in ("container_id", "hostname", "hostip", "port")): + if len(self.container_chaining) == 0 or not all(k in self.container_chaining[0] for k in ("container_id", "hostname", "hostip", "port")): raise exceptions.MoonError('Void container chaining') self.pdp_container = self.container_chaining[0]["container_id"] diff --git a/moon_interface/moon_interface/http_server.py b/moon_interface/moon_interface/http_server.py index 82ee5d91..a2e25377 100644 --- a/moon_interface/moon_interface/http_server.py +++ b/moon_interface/moon_interface/http_server.py @@ -114,12 +114,6 @@ class HTTPServer(Server): def get_400_json(e): return jsonify({"result": False, "code": 400, "description": str(e)}), 400 - ''' - [Note] i have tried to simulate authz post request (authz_Requests/run) to return 500 response code - and an AuthzException thrown from their [Line 63] and catched here , then the server here return response - with 403 code [Forbidden] , is it correct if so why sometime at authz [from Line 137] return a response - with error code , i think we can do it in the same way as the one mentioned? - ''' self.app.register_error_handler(400, lambda e: get_400_json) self.app.register_error_handler(403, exceptions.AuthException) diff --git a/moon_interface/moon_interface/server.py b/moon_interface/moon_interface/server.py index d38dae28..29403a6b 100644 --- a/moon_interface/moon_interface/server.py +++ b/moon_interface/moon_interface/server.py @@ -16,9 +16,7 @@ def create_server(): conf = configuration.get_configuration("components/pipeline").get( "components/pipeline", {}).get("interface", {}) - ''' - [Note] i think pipeline should be changed to interface - ''' + hostname = conf.get("hostname", "pipeline") port = conf.get("port", 80) bind = conf.get("bind", "127.0.0.1") @@ -26,9 +24,7 @@ def create_server(): hostname = "interface" bind = "127.0.0.1" port = 80 - ''' - [Note] i think pipeline should be changed to interface - ''' + configuration.add_component(uuid="pipeline", name=hostname, port=port, diff --git a/moon_manager/moon_manager/__init__.py b/moon_manager/moon_manager/__init__.py index 6f964a63..85c245e0 100644 --- a/moon_manager/moon_manager/__init__.py +++ b/moon_manager/moon_manager/__init__.py @@ -3,4 +3,4 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -__version__ = "4.3.2" +__version__ = "4.4.0" diff --git a/moon_manager/moon_manager/api/slaves.py b/moon_manager/moon_manager/api/slaves.py new file mode 100644 index 00000000..f5b3fa14 --- /dev/null +++ b/moon_manager/moon_manager/api/slaves.py @@ -0,0 +1,110 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. +""" +PDP are Policy Decision Point. + +""" + +from flask import request +from flask_restful import Resource +import logging +import requests +import time +from python_moonutilities.security_functions import check_auth +from python_moondb.core import PDPManager +from python_moondb.core import PolicyManager +from python_moondb.core import ModelManager +from python_moonutilities import configuration, exceptions + +__version__ = "4.3.0" + +logger = logging.getLogger("moon.manager.api." + __name__) + + +class Slaves(Resource): + """ + Endpoint for pdp requests + """ + + __urls__ = ( + "/slaves", + "/slaves/", + "/slaves/<string:uuid>", + "/slaves/<string:uuid>/", + ) + + def __init__(self, **kwargs): + conf = configuration.get_configuration("components/orchestrator") + self.orchestrator_hostname = conf["components/orchestrator"].get("hostname", + "orchestrator") + self.orchestrator_port = conf["components/orchestrator"].get("port", + 80) + + @check_auth + def get(self, uuid=None, user_id=None): + """Retrieve all slaves + + :param uuid: uuid of the slave + :param user_id: user ID who do the request + :return: { + "slaves": { + "XXX": { + "name": "...", + "installed": True + }, + "YYY": { + "name": "...", + "installed": False + } + } + } + """ + req = requests.get("http://{}:{}/slaves".format( + self.orchestrator_hostname, self.orchestrator_port + )) + return {"slaves": req.json().get("slaves", dict())} + + @check_auth + def patch(self, uuid=None, user_id=None): + """Update a slave + + :param uuid: uuid of the slave to update + :param user_id: user ID who do the request + :request body: { + "op": "replace", + "variable": "configured", + "value": True, + } + :return: 204 + :internal_api: add_pdp + """ + logger.info("Will made a request for {}".format(uuid)) + if request.json.get("op") == "replace" \ + and request.json.get("variable") == "configured" \ + and request.json.get("value"): + req = requests.post("http://{}:{}/pods".format( + self.orchestrator_hostname, self.orchestrator_port, + ), + json={"slave_name": uuid} + ) + if req.status_code != 200: + logger.warning("Get error from Orchestrator {} {}".format( + req.reason, req.status_code + )) + return "Orchestrator: " + str(req.reason), req.status_code + elif request.json.get("op") == "replace" \ + and request.json.get("variable") == "configured" \ + and not request.json.get("value"): + req = requests.delete("http://{}:{}/pods/{}".format( + self.orchestrator_hostname, self.orchestrator_port, uuid + )) + if req.status_code != 200: + logger.warning("Get error from Orchestrator {} {}".format( + req.reason, req.status_code + )) + return "Orchestrator: " + str(req.reason), req.status_code + else: + return "Malformed request", 400 + return {"slaves": req.json()} diff --git a/moon_manager/moon_manager/http_server.py b/moon_manager/moon_manager/http_server.py index d67e1121..a98cab43 100644 --- a/moon_manager/moon_manager/http_server.py +++ b/moon_manager/moon_manager/http_server.py @@ -14,6 +14,7 @@ from moon_manager.api.generic import Status, Logs, API from moon_manager.api.models import Models from moon_manager.api.policies import Policies from moon_manager.api.pdp import PDP +from moon_manager.api.slaves import Slaves from moon_manager.api.meta_rules import MetaRules from moon_manager.api.meta_data import SubjectCategories, ObjectCategories, ActionCategories from moon_manager.api.perimeter import Subjects, Objects, Actions @@ -32,7 +33,7 @@ __API__ = ( Subjects, Objects, Actions, Rules, SubjectAssignments, ObjectAssignments, ActionAssignments, SubjectData, ObjectData, ActionData, - Models, Policies, PDP + Models, Policies, PDP, Slaves ) diff --git a/moon_manager/tests/functional_pod/conftest.py b/moon_manager/tests/functional_pod/conftest.py new file mode 100644 index 00000000..b5811755 --- /dev/null +++ b/moon_manager/tests/functional_pod/conftest.py @@ -0,0 +1,12 @@ +import pytest + +print("ANALYSING CONFTEST") + + +@pytest.fixture +def context(): + print("CREATING CONTEXT") + yield { + "hostname": "manager", + "port": 8082, + } diff --git a/moon_manager/tests/functional_pod/run_functional_tests.sh b/moon_manager/tests/functional_pod/run_functional_tests.sh index c80bf15d..7a95a491 100644 --- a/moon_manager/tests/functional_pod/run_functional_tests.sh +++ b/moon_manager/tests/functional_pod/run_functional_tests.sh @@ -1,11 +1,4 @@ #!/usr/bin/env bash -set -x - -kubectl create -n moon -f tools/moon_kubernetes/templates/moon_forming.yaml - -echo Waiting for jobs forming -sleep 5 -kubectl get jobs -n moon -kubectl logs -n moon jobs/forming - +cd /data/tests/functional_pod +pytest . diff --git a/moon_manager/tests/functional_pod/test_manager.py b/moon_manager/tests/functional_pod/test_manager.py new file mode 100644 index 00000000..aab5fba4 --- /dev/null +++ b/moon_manager/tests/functional_pod/test_manager.py @@ -0,0 +1,77 @@ +import json +import requests + + +def get_json(data): + return json.loads(data.decode("utf-8")) + + +def get_pdp(context): + req = requests.get("http://{}:{}/pdp".format( + context.get("hostname"), + context.get("port")), + timeout=3) + pdp = req.json() + return req, pdp + + +def add_pdp(context, data): + req = requests.post("http://{}:{}/pdp".format( + context.get("hostname"), + context.get("port")), + data=json.dumps(data), + headers={'Content-Type': 'application/json'}, + timeout=3) + pdp = req.json() + return req, pdp + + +def delete_pdp(context, key): + req = requests.delete("http://{}:{}/pdp/{}".format( + context.get("hostname"), + context.get("port"), key), + timeout=3) + return req + + +def delete_pdp_without_id(context): + req = requests.delete("http://{}:{}/pdp/{}".format( + context.get("hostname"), + context.get("port"), ""), + timeout=3) + return req + + +def test_get_pdp(context): + req, pdp = get_pdp(context) + assert req.status_code == 200 + assert isinstance(pdp, dict) + assert "pdps" in pdp + + +def test_add_pdp(context): + data = { + "name": "testuser", + "security_pipeline": ["policy_id_1", "policy_id_2"], + "keystone_project_id": "keystone_project_id", + "description": "description of testuser" + } + req, pdp = add_pdp(context, data) + assert req.status_code == 200 + assert isinstance(pdp, dict) + value = list(pdp["pdps"].values())[0] + assert "pdps" in pdp + assert value['name'] == "testuser" + assert value["description"] == "description of {}".format("testuser") + assert value["keystone_project_id"] == "keystone_project_id" + + +def test_delete_pdp(context): + request, pdp = get_pdp(context) + success_req = None + for key, value in pdp['pdps'].items(): + if value['name'] == "testuser": + success_req = delete_pdp(context, key) + break + assert success_req + assert success_req.status_code == 200 diff --git a/moon_manager/tests/functional_pod/test_models.py b/moon_manager/tests/functional_pod/test_models.py new file mode 100644 index 00000000..dcda9f32 --- /dev/null +++ b/moon_manager/tests/functional_pod/test_models.py @@ -0,0 +1,78 @@ +import json +import requests + + +def get_models(context): + req = requests.get("http://{}:{}/models".format( + context.get("hostname"), + context.get("port")), + timeout=3) + models = req.json() + return req, models + + +def add_models(context, name): + data = { + "name": name, + "description": "description of {}".format(name), + "meta_rules": ["meta_rule_id1", "meta_rule_id2"] + } + req = requests.post("http://{}:{}/models".format( + context.get("hostname"), + context.get("port")), + data=json.dumps(data), + headers={'Content-Type': 'application/json'}, + timeout=3) + models = req.json() + return req, models + + +def delete_models(context, name): + _, models = get_models(context) + request = None + for key, value in models['models'].items(): + if value['name'] == name: + request = requests.delete("http://{}:{}/models/{}".format(key, + context.get("hostname"), + context.get("port")), + timeout=3) + break + return request + + +def delete_models_without_id(context): + req = requests.delete("http://{}:{}/models/{}".format( + context.get("hostname"), + context.get("port"), + ""), + timeout=3) + return req + + +def test_get_models(context): + req, models = get_models(context) + assert req.status_code == 200 + assert isinstance(models, dict) + assert "models" in models + + +def test_add_models(context): + req, models = add_models(context, "testuser") + assert req.status_code == 200 + assert isinstance(models, dict) + value = list(models["models"].values())[0] + assert "models" in models + assert value['name'] == "testuser" + assert value["description"] == "description of {}".format("testuser") + assert value["meta_rules"][0] == "meta_rule_id1" + + +def test_delete_models(context): + req = delete_models(context, "testuser") + assert req.status_code == 200 + + +def test_delete_models_without_id(context): + req = delete_models_without_id(context) + assert req.status_code == 500 + diff --git a/moon_manager/tests/unit_python/api/test_assignemnt.py b/moon_manager/tests/unit_python/api/test_assignemnt.py new file mode 100644 index 00000000..08688e04 --- /dev/null +++ b/moon_manager/tests/unit_python/api/test_assignemnt.py @@ -0,0 +1,174 @@ +import api.utilities as utilities +import json + + +# subject_categories_test + + +def get_subject_assignment(client, policy_id): + req = client.get("/policies/{}/subject_assignments".format(policy_id)) + subject_assignment = utilities.get_json(req.data) + return req, subject_assignment + + +def add_subject_assignment(client, policy_id, category_id): + data = { + "id": "id1", + "category_id": category_id, + "data_id": "data_id1" + } + req = client.post("/policies/{}/subject_assignments/{}".format(policy_id, category_id), data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + subject_assignment = utilities.get_json(req.data) + return req, subject_assignment + + +def delete_subject_assignment(client, policy_id): + req = client.delete("/policies/{}/subject_assignments".format(policy_id)) + return req + + +def test_get_subject_assignment(): + policy_id = utilities.get_policy_id() + client = utilities.register_client() + req, subject_assignment = get_subject_assignment(client, policy_id) + assert req.status_code == 200 + assert isinstance(subject_assignment, dict) + assert "subject_assignments" in subject_assignment + + +def test_add_subject_assignment(): + policy_id = utilities.get_policy_id() + client = utilities.register_client() + req, subject_assignment = add_subject_assignment(client, policy_id, "111") + assert req.status_code == 200 + assert isinstance(subject_assignment, dict) + value = subject_assignment["subject_assignments"] + assert "subject_assignments" in subject_assignment + id = list(value.keys())[0] + assert value[id]['policy_id'] == policy_id + assert value[id]['category_id'] == "111" + assert value[id]['subject_id'] == "id1" + + +def test_delete_subject_assignment(): + client = utilities.register_client() + policy_id = utilities.get_policy_id() + success_req = delete_subject_assignment(client, policy_id) + assert success_req.status_code == 200 + +# --------------------------------------------------------------------------- + +# object_categories_test + + +def get_object_assignment(client, policy_id): + req = client.get("/policies/{}/object_assignments".format(policy_id)) + object_assignment = utilities.get_json(req.data) + return req, object_assignment + + +def add_object_assignment(client, policy_id, category_id): + data = { + "id": "id1", + "category_id": category_id, + "data_id": "data_id1" + } + req = client.post("/policies/{}/object_assignments/{}".format(policy_id, category_id), data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + object_assignment = utilities.get_json(req.data) + return req, object_assignment + + +def delete_object_assignment(client, policy_id): + req = client.delete("/policies/{}/object_assignments".format(policy_id)) + return req + + +def test_get_object_assignment(): + policy_id = utilities.get_policy_id() + client = utilities.register_client() + req, object_assignment = get_object_assignment(client, policy_id) + assert req.status_code == 200 + assert isinstance(object_assignment, dict) + assert "object_assignments" in object_assignment + + +def test_add_object_assignment(): + policy_id = utilities.get_policy_id() + client = utilities.register_client() + req, object_assignment = add_object_assignment(client, policy_id, "111") + assert req.status_code == 200 + assert isinstance(object_assignment, dict) + value = object_assignment["object_assignments"] + assert "object_assignments" in object_assignment + id = list(value.keys())[0] + assert value[id]['policy_id'] == policy_id + assert value[id]['category_id'] == "111" + assert value[id]['object_id'] == "id1" + + +def test_delete_object_assignment(): + client = utilities.register_client() + policy_id = utilities.get_policy_id() + success_req = delete_object_assignment(client, policy_id) + assert success_req.status_code == 200 + +# --------------------------------------------------------------------------- + +# action_categories_test + + +def get_action_assignment(client, policy_id): + req = client.get("/policies/{}/action_assignments".format(policy_id)) + action_assignment = utilities.get_json(req.data) + return req, action_assignment + + +def add_action_assignment(client, policy_id, category_id): + data = { + "id": "id1", + "category_id": category_id, + "data_id": "data_id1" + } + req = client.post("/policies/{}/action_assignments/{}".format(policy_id, category_id), data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + action_assignment = utilities.get_json(req.data) + return req, action_assignment + + +def delete_action_assignment(client, policy_id): + req = client.delete("/policies/{}/action_assignments".format(policy_id)) + return req + + +def test_get_action_assignment(): + policy_id = utilities.get_policy_id() + client = utilities.register_client() + req, action_assignment = get_action_assignment(client, policy_id) + assert req.status_code == 200 + assert isinstance(action_assignment, dict) + assert "action_assignments" in action_assignment + + +def test_add_action_assignment(): + policy_id = utilities.get_policy_id() + client = utilities.register_client() + req, action_assignment = add_action_assignment(client, policy_id, "111") + assert req.status_code == 200 + assert isinstance(action_assignment, dict) + value = action_assignment["action_assignments"] + assert "action_assignments" in action_assignment + id = list(value.keys())[0] + assert value[id]['policy_id'] == policy_id + assert value[id]['category_id'] == "111" + assert value[id]['action_id'] == "id1" + + +def test_delete_action_assignment(): + client = utilities.register_client() + policy_id = utilities.get_policy_id() + success_req = delete_action_assignment(client, policy_id) + assert success_req.status_code == 200 + +# ---------------------------------------------------------------------------
\ No newline at end of file diff --git a/moon_manager/tests/unit_python/api/test_rules.py b/moon_manager/tests/unit_python/api/test_rules.py new file mode 100644 index 00000000..86a3d390 --- /dev/null +++ b/moon_manager/tests/unit_python/api/test_rules.py @@ -0,0 +1,58 @@ +import api.utilities as utilities +import json + + +def get_rules(client, policy_id): + req = client.get("/policies/{}/rules".format(policy_id)) + rules = utilities.get_json(req.data) + return req, rules + + +def add_rules(client, policy_id): + data = { + "meta_rule_id": "meta_rule_id1", + "rule": ["subject_data_id2", "object_data_id2", "action_data_id2"], + "instructions": ( + {"decision": "grant"}, + ), + "enabled": True + } + req = client.post("/policies/{}/rules".format(policy_id), data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + rules = utilities.get_json(req.data) + return req, rules + + +def delete_rules(client, policy_id, meta_rule_id): + req = client.delete("/policies/{}/rules/{}".format(policy_id, meta_rule_id)) + return req + + +def test_get_rules(): + policy_id = utilities.get_policy_id() + client = utilities.register_client() + req, rules = get_rules(client, policy_id) + assert req.status_code == 200 + assert isinstance(rules, dict) + assert "rules" in rules + + +def test_add_rules(): + policy_id = utilities.get_policy_id() + client = utilities.register_client() + req, rules = add_rules(client, policy_id) + assert req.status_code == 200 + assert isinstance(rules, dict) + value = rules["rules"] + assert "rules" in rules + id = list(value.keys())[0] + assert value[id]["meta_rule_id"] == "meta_rule_id1" + + +def test_delete_rules(): + client = utilities.register_client() + policy_id = utilities.get_policy_id() + req, added_rules = get_rules(client, policy_id) + id = added_rules["rules"]['rules'][0]['id'] + rules = delete_rules(client, policy_id, id) + assert rules.status_code == 200 diff --git a/moon_orchestrator/moon_orchestrator/__init__.py b/moon_orchestrator/moon_orchestrator/__init__.py index 3276f42d..85c245e0 100644 --- a/moon_orchestrator/moon_orchestrator/__init__.py +++ b/moon_orchestrator/moon_orchestrator/__init__.py @@ -3,4 +3,4 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -__version__ = "4.3.1" +__version__ = "4.4.0" diff --git a/moon_orchestrator/moon_orchestrator/api/pods.py b/moon_orchestrator/moon_orchestrator/api/pods.py index 31ae31de..3a01c3a9 100644 --- a/moon_orchestrator/moon_orchestrator/api/pods.py +++ b/moon_orchestrator/moon_orchestrator/api/pods.py @@ -6,6 +6,7 @@ from flask import request from flask_restful import Resource from python_moonutilities.security_functions import check_auth +from python_moonutilities import exceptions import logging logger = logging.getLogger("moon.orchestrator.api.pods") @@ -17,6 +18,7 @@ class Pods(Resource): """ __version__ = "4.3.1" + POD_TYPES = ("authz", "wrapper") __urls__ = ( "/pods", @@ -57,6 +59,21 @@ class Pods(Resource): except Exception as e: return {"result": False, "message": str(e)}, 500 + def __get_pod_with_keystone_pid(self, keystone_pid): + for pod_key, pod_values in self.driver.get_pods().items(): + if pod_values[0]['keystone_project_id'] == keystone_pid: + return True + + def __get_wrapper(self, slave_name): + for slave in self.driver.get_slaves(): + if slave_name == slave["name"] \ + and slave["configured"]: + return True + + def __get_slave_names(self): + for slave in self.driver.get_slaves(): + yield slave["name"] + @check_auth def post(self, uuid=None, user_id=None): """Create a new pod. @@ -64,9 +81,14 @@ class Pods(Resource): :param uuid: uuid of the pod (not used here) :param user_id: user ID who do the request :request body: { - "name": "...", - "description": "...", - "type": "plugin_name" + "pdp_id": "fa2323f7055d4a88b1b85d31fe5e8369", + "name": "pdp_rbac3", + "keystone_project_id": "ceacbb5564cc48ad929dd4f00e52bf63", + "models": {...}, + "policies": {...}, + "description": "test", + "security_pipeline": [...], + "slave_name": "" } :return: { "pdp_id1": { @@ -76,25 +98,35 @@ class Pods(Resource): } } """ - logger.debug("POST param={}".format(request.json)) - try: + pods = {} + if "security_pipeline" in request.json: + if self.__get_pod_with_keystone_pid(request.json.get("keystone_project_id")): + raise exceptions.PipelineConflict self.driver.create_pipeline( request.json.get("keystone_project_id"), request.json.get("pdp_id"), request.json.get("security_pipeline"), manager_data=request.json, - active_context=None, - active_context_name=None) - pods = {} + slave_name=request.json.get("slave_name")) for _pod_key, _pod_values in self.driver.get_pods().items(): pods[_pod_key] = [] for _pod_value in _pod_values: if _pod_value['namespace'] != "moon": continue pods[_pod_key].append(_pod_value) - return {"pods": pods} - except Exception as e: - return {"result": False, "message": str(e)}, 500 + else: + logger.info("------------------------------------") + logger.info(list(self.__get_slave_names())) + logger.info("------------------------------------") + if self.__get_wrapper(request.json.get("slave_name")): + raise exceptions.WrapperConflict + if request.json.get("slave_name") not in self.__get_slave_names(): + raise exceptions.SlaveNameUnknown + slave_name = request.json.get("slave_name") + if not slave_name: + slave_name = self.driver.get_slaves(active=True) + self.driver.create_wrappers(slave_name) + return {"pods": self.driver.get_pods()} @check_auth def delete(self, uuid=None, user_id=None): @@ -110,27 +142,31 @@ class Pods(Resource): try: self.driver.delete_pipeline(uuid) return {'result': True} + except exceptions.PipelineUnknown: + for slave in self.driver.get_slaves(): + if uuid in (slave['name'], slave["wrapper_name"]): + self.driver.delete_wrapper(name=slave["wrapper_name"]) except Exception as e: return {"result": False, "message": str(e)}, 500 - @check_auth - def patch(self, uuid=None, user_id=None): - """Update a pod - - :param uuid: uuid of the pdp to update - :param user_id: user ID who do the request - :request body: { - "name": "...", - "replicas": "...", - "description": "...", - } - :return: { - "pod_id1": { - "name": "...", - "replicas": "...", - "description": "...", - } - } - :internal_api: update_pdp - """ - return {"pods": None} + # @check_auth + # def patch(self, uuid=None, user_id=None): + # """Update a pod + # + # :param uuid: uuid of the pdp to update + # :param user_id: user ID who do the request + # :request body: { + # "name": "...", + # "replicas": "...", + # "description": "...", + # } + # :return: { + # "pod_id1": { + # "name": "...", + # "replicas": "...", + # "description": "...", + # } + # } + # :internal_api: update_pdp + # """ + # return {"pods": None} diff --git a/moon_orchestrator/moon_orchestrator/api/slaves.py b/moon_orchestrator/moon_orchestrator/api/slaves.py new file mode 100644 index 00000000..7453d305 --- /dev/null +++ b/moon_orchestrator/moon_orchestrator/api/slaves.py @@ -0,0 +1,46 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +from flask import request +from flask_restful import Resource +from python_moonutilities.security_functions import check_auth +import logging + +logger = logging.getLogger("moon.orchestrator.api.slaves") + + +class Slaves(Resource): + """ + Endpoint for slaves requests + """ + + __version__ = "4.3.1" + + __urls__ = ( + "/slaves", + "/slaves/", + "/slaves/<string:uuid>", + "/slaves/<string:uuid>/", + ) + + def __init__(self, **kwargs): + self.driver = kwargs.get("driver") + + @check_auth + def get(self, uuid=None, user_id=None): + """Retrieve all pods + + :param uuid: uuid of the pod + :param user_id: user ID who do the request + :return: { + "pod_id1": { + "name": "...", + "replicas": "...", + "description": "...", + } + } + """ + slaves = self.driver.get_slaves() + return {"slaves": slaves} diff --git a/moon_orchestrator/moon_orchestrator/drivers.py b/moon_orchestrator/moon_orchestrator/drivers.py index 9c230cce..bb0d0c2c 100644 --- a/moon_orchestrator/moon_orchestrator/drivers.py +++ b/moon_orchestrator/moon_orchestrator/drivers.py @@ -159,17 +159,17 @@ class K8S(Driver): resp = client.create_namespaced_service(namespace="moon", body=service_manifest) logger.info("Service {} created!".format(data.get('name'))) - return resp + return service_manifest def load_deployment_and_service(self, data, api_client=None, ext_client=None, expose=False): _client = api_client if api_client else self.client + manifest = self.__create_service(client=_client, data=data[0], + expose=expose) + data[0]["external_port"] = manifest['spec']['ports'][0].get('nodePort') pod = self.__create_deployment(client=ext_client, data=data) - self.__create_service(client=_client, data=data[0], - expose=expose) self.cache[pod.metadata.uid] = data - @staticmethod - def delete_deployment(name=None, namespace="moon", ext_client=None): + def delete_deployment(self, name=None, namespace="moon", ext_client=None): logger.info("Deleting deployment {}".format(name)) body = client.V1DeleteOptions(propagation_policy='Foreground') ret = ext_client.delete_namespaced_deployment( @@ -177,7 +177,16 @@ class K8S(Driver): namespace=namespace, body=body ) - logger.info(ret) + logger.debug(ret) + _uid = None + for uid, value in self.cache.items(): + if value[0]['name'] == name: + _uid = uid + break + if _uid: + self.cache.pop(_uid) + else: + raise exceptions.DockerError("Cannot find and delete pod named {}".format(name)) def delete_service(self, name, namespace="moon", api_client=None): if not api_client: @@ -185,12 +194,45 @@ class K8S(Driver): ret = api_client.delete_namespaced_service(name=name, namespace=namespace) logger.debug("delete_service {}".format(ret)) - def get_slaves(self): + def get_slaves(self, active=False): + contexts, active_context = self.get_contexts() + pods = self.get_pods() + # logger.info("pods = {}".format(pods)) + slaves = [] + if active: + for key, value in pods.items(): + # logger.info("ctx={}".format(active_context)) + # logger.info("value={}".format(value)) + if active_context["name"] == value[0].get('slave_name'): + data = dict(active_context) + data["wrapper_name"] = value[0]['name'] + data["ip"] = value[0].get("ip", "NC") + data["port"] = value[0].get("external_port", "NC") + slaves.append(data) + break + return slaves + for ctx in contexts: + data = dict(ctx) + data["configured"] = False + for key, value in pods.items(): + # logger.info("ctx={}".format(ctx)) + # logger.info("value={}".format(value)) + if ctx["name"] == value[0].get('slave_name'): + data["wrapper_name"] = value[0]['name'] + data["ip"] = value[0].get("ip", "NC") + data["port"] = value[0].get("external_port", "NC") + data["configured"] = True + break + slaves.append(data) + return slaves + + @staticmethod + def get_contexts(): contexts, active_context = config.list_kube_config_contexts() return contexts, active_context - def create_wrappers(self): - contexts, active_context = self.get_slaves() + def create_wrappers(self, slave_name=None): + contexts, active_context = self.get_contexts() logger.debug("contexts: {}".format(contexts)) logger.debug("active_context: {}".format(active_context)) conf = configuration.get_configuration("components/wrapper") @@ -201,6 +243,8 @@ class K8S(Driver): "container", "wukongsun/moon_wrapper:v4.3") for _ctx in contexts: + if slave_name and slave_name != _ctx['name']: + continue _config = config.new_client_from_config(context=_ctx['name']) logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) @@ -209,14 +253,56 @@ class K8S(Driver): "name": hostname + "-" + get_random_name(), "container": container, "port": port, - "namespace": "moon" + "namespace": "moon", + "slave_name": _ctx['name'] }, ] self.load_deployment_and_service(data, api_client, ext_client, expose=True) + def delete_wrapper(self, uuid=None, name=None, namespace="moon", + active_context=None, + active_context_name=None): + name_to_delete = None + if uuid and uuid in self.get_pods(): + name_to_delete = self.get_pods()[uuid][0]['name'] + elif name: + for pod_key, pod_list in self.get_pods().items(): + for pod_value in pod_list: + if pod_value.get("name") == name: + name_to_delete = pod_value.get("name") + break + if not name_to_delete: + raise exceptions.WrapperUnknown + contexts, _active_context = self.get_contexts() + if active_context_name: + for _context in contexts: + if _context["name"] == active_context_name: + active_context = _context + break + if active_context: + active_context = _active_context + _config = config.new_client_from_config( + context=active_context['name']) + logger.debug("_config={}".format(_config)) + api_client = client.CoreV1Api(_config) + ext_client = client.ExtensionsV1beta1Api(_config) + self.delete_deployment(name=name_to_delete, namespace=namespace, + ext_client=ext_client) + self.delete_service(name=name_to_delete, api_client=api_client) + return + logger.debug("contexts={}".format(contexts)) + for _ctx in contexts: + _config = config.new_client_from_config(context=_ctx['name']) + logger.debug("_config={}".format(_config)) + api_client = client.CoreV1Api(_config) + ext_client = client.ExtensionsV1beta1Api(_config) + self.delete_deployment(name=name_to_delete, namespace=namespace, + ext_client=ext_client) + self.delete_service(name=name_to_delete, api_client=api_client) + def create_pipeline(self, keystone_project_id, pdp_id, policy_ids, manager_data=None, active_context=None, - active_context_name=None): + slave_name=None): """ Create security functions :param keystone_project_id: the Keystone project id @@ -225,7 +311,7 @@ class K8S(Driver): :param manager_data: data needed to create pods :param active_context: if present, add the security function in this context - :param active_context_name: if present, add the security function in + :param slave_name: if present, add the security function in this context name if active_context_name and active_context are not present, add the security function in all context (ie, in all slaves) @@ -295,12 +381,12 @@ class K8S(Driver): "namespace": "moon" }) logger.debug("data={}".format(data)) - contexts, _active_context = self.get_slaves() - logger.debug("active_context_name={}".format(active_context_name)) + contexts, _active_context = self.get_contexts() + logger.debug("active_context_name={}".format(slave_name)) logger.debug("active_context={}".format(active_context)) - if active_context_name: + if slave_name: for _context in contexts: - if _context["name"] == active_context_name: + if _context["name"] == slave_name: active_context = _context break if active_context: @@ -314,6 +400,8 @@ class K8S(Driver): return logger.debug("contexts={}".format(contexts)) for _ctx in contexts: + if slave_name and slave_name != _ctx['name']: + continue _config = config.new_client_from_config(context=_ctx['name']) logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) @@ -342,9 +430,9 @@ class K8S(Driver): name_to_delete = pod_value.get("name") break if not name_to_delete: - raise exceptions.MoonError("Cannot find pipeline") + raise exceptions.PipelineUnknown logger.info("Will delete deployment and service named {}".format(name_to_delete)) - contexts, _active_context = self.get_slaves() + contexts, _active_context = self.get_contexts() if active_context_name: for _context in contexts: if _context["name"] == active_context_name: diff --git a/moon_orchestrator/moon_orchestrator/http_server.py b/moon_orchestrator/moon_orchestrator/http_server.py index fa5308d0..85e29cd0 100644 --- a/moon_orchestrator/moon_orchestrator/http_server.py +++ b/moon_orchestrator/moon_orchestrator/http_server.py @@ -10,6 +10,7 @@ import requests import time from moon_orchestrator import __version__ from moon_orchestrator.api.pods import Pods +from moon_orchestrator.api.slaves import Slaves from moon_orchestrator.api.generic import Status from moon_orchestrator.drivers import get_driver from python_moonutilities import configuration, exceptions @@ -122,7 +123,7 @@ class HTTPServer(Server): if "pdps" in pdp.json(): break logger.debug("pdp={}".format(pdp)) - self.driver.create_wrappers() + # self.driver.create_wrappers() for _pdp_key, _pdp_value in pdp.json()['pdps'].items(): if _pdp_value.get('keystone_project_id'): # TODO: select context to add security function @@ -151,6 +152,10 @@ class HTTPServer(Server): resource_class_kwargs={ "driver": self.driver }) + self.api.add_resource(Slaves, *Slaves.__urls__, + resource_class_kwargs={ + "driver": self.driver + }) def run(self): self.app.run(host=self._host, port=self._port) # nosec diff --git a/moon_orchestrator/tests/unit_python/test_pods.py b/moon_orchestrator/tests/unit_python/test_pods.py index f760aa62..678645be 100644 --- a/moon_orchestrator/tests/unit_python/test_pods.py +++ b/moon_orchestrator/tests/unit_python/test_pods.py @@ -31,7 +31,7 @@ def test_get_pods_failure(context, monkeypatch): assert not data["pods"] -def test_add_pods(context, monkeypatch): +def test_add_pods_with_pipeline(context, monkeypatch): patch_k8s(monkeypatch) import moon_orchestrator.server @@ -52,6 +52,60 @@ def test_add_pods(context, monkeypatch): assert data["pods"] +def test_add_pods_without_pipeline_with_bad_slave_name(context, monkeypatch): + patch_k8s(monkeypatch) + + import moon_orchestrator.server + server = moon_orchestrator.server.create_server() + _client = server.app.test_client() + data = { + "slave_name": "test", + } + req = _client.post("/pods", data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + assert req.status_code == 400 + assert req.data + data = get_json(req.data) + assert isinstance(data, dict) + assert 'The slave is unknown.' in data['message'] + + +def test_add_pods_without_pipeline_with_good_slave_name(context, monkeypatch): + patch_k8s(monkeypatch) + + import moon_orchestrator.server + server = moon_orchestrator.server.create_server() + _client = server.app.test_client() + data = { + "slave_name": "active_context", + } + req = _client.post("/pods", data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + assert req.status_code == 200 + assert req.data + data = get_json(req.data) + assert isinstance(data, dict) + assert "pods" in data + assert data["pods"] + + +def test_add_pods_without_pipeline_without_slave_name(context, monkeypatch): + patch_k8s(monkeypatch) + + import moon_orchestrator.server + server = moon_orchestrator.server.create_server() + _client = server.app.test_client() + data = { + } + req = _client.post("/pods", data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + assert req.status_code == 400 + assert req.data + data = get_json(req.data) + assert isinstance(data, dict) + assert 'The slave is unknown.' in data['message'] + + def test_add_pods_with_no_data(context, monkeypatch): patch_k8s(monkeypatch) import moon_orchestrator.server @@ -59,10 +113,10 @@ def test_add_pods_with_no_data(context, monkeypatch): _client = server.app.test_client() req = _client.post("/pods", data=json.dumps({}), headers={'Content-Type': 'application/json'}) - assert req.status_code == 500 + assert req.status_code == 400 assert req.data data = get_json(req.data) - assert '400: Policy Unknown' in data['message'] + assert 'The slave is unknown.' in data['message'] def test_add_pods_with_no_policies_no_models(context, monkeypatch, no_requests): diff --git a/moon_orchestrator/tests/unit_python/test_slaves.py b/moon_orchestrator/tests/unit_python/test_slaves.py new file mode 100644 index 00000000..88ff7e55 --- /dev/null +++ b/moon_orchestrator/tests/unit_python/test_slaves.py @@ -0,0 +1,17 @@ +import json +from mock_pods import patch_k8s +from utilities import get_json + + +def test_get_slaves(context, monkeypatch): + patch_k8s(monkeypatch) + + import moon_orchestrator.server + server = moon_orchestrator.server.create_server() + _client = server.app.test_client() + req = _client.get("/slaves") + assert req.status_code == 200 + assert req.data + data = get_json(req.data) + assert isinstance(data, dict) + assert "slaves" in data diff --git a/python_moonclient/Changelog b/python_moonclient/Changelog index f6f6c3a4..64ae76ba 100644 --- a/python_moonclient/Changelog +++ b/python_moonclient/Changelog @@ -30,3 +30,10 @@ CHANGES - moon_create_pdp - moon_send_authz_to_wrapper - Fix a bug in pdp library + +1.2.0 +----- +- Add some commands: + - moon_get_slaves + - moon_set_slave + - moon_delete_slave diff --git a/python_moonclient/python_moonclient/__init__.py b/python_moonclient/python_moonclient/__init__.py index 2302dea9..2c7f8f5c 100644 --- a/python_moonclient/python_moonclient/__init__.py +++ b/python_moonclient/python_moonclient/__init__.py @@ -3,4 +3,4 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -__version__ = "1.1.0" +__version__ = "1.2.0" diff --git a/python_moonclient/python_moonclient/scripts.py b/python_moonclient/python_moonclient/scripts.py index c880e497..74ed47fc 100644 --- a/python_moonclient/python_moonclient/scripts.py +++ b/python_moonclient/python_moonclient/scripts.py @@ -1,6 +1,6 @@ import logging from importlib.machinery import SourceFileLoader -from . import parse, models, policies, pdp, authz +from . import parse, models, policies, pdp, authz, slaves logger = logging.getLogger("moonclient.scripts") @@ -161,3 +161,75 @@ def map_pdp_to_project(): logger.info("Mapping: {}=>{}".format(args.filename[0], args.filename[1])) # TODO: check if pdp_id and keystone_project_id exist pdp.map_to_keystone(pdp_id=args.filename[0], keystone_project_id=args.filename[1]) + + +def get_slaves(): + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.WARNING) + requests_log.propagate = True + + args = parse.parse() + consul_host = args.consul_host + consul_port = args.consul_port + + models.init(consul_host, consul_port) + policies.init(consul_host, consul_port) + pdp.init(consul_host, consul_port) + slaves.init(consul_host, consul_port) + + for value in slaves.get_slaves().get('slaves', dict()): + if value['configured']: + print(" {} (configured)".format(value['name'])) + else: + print(" {} (not configured)".format(value['name'])) + + +def set_slave(): + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.WARNING) + requests_log.propagate = True + + args = parse.parse() + consul_host = args.consul_host + consul_port = args.consul_port + + models.init(consul_host, consul_port) + policies.init(consul_host, consul_port) + pdp.init(consul_host, consul_port) + slaves.init(consul_host, consul_port) + + slave_name = "kubernetes-admin@kubernetes" + if args.filename: + slave_name = args.filename + for value in slaves.set_slave(slave_name).get('slaves', dict()): + if value['configured']: + print(" {} (configured)".format(value['name'])) + else: + print(" {} (not configured)".format(value['name'])) + + +def delete_slave(): + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.WARNING) + requests_log.propagate = True + + args = parse.parse() + consul_host = args.consul_host + consul_port = args.consul_port + + models.init(consul_host, consul_port) + policies.init(consul_host, consul_port) + pdp.init(consul_host, consul_port) + slaves.init(consul_host, consul_port) + + slave_name = "kubernetes-admin@kubernetes" + if args.filename: + slave_name = args.filename + for value in slaves.delete_slave(slave_name).get('slaves', dict()): + if value['configured']: + print(" {} (configured)".format(value['name'])) + else: + print(" {} (not configured)".format(value['name'])) + + + diff --git a/python_moonclient/python_moonclient/slaves.py b/python_moonclient/python_moonclient/slaves.py new file mode 100644 index 00000000..3554341d --- /dev/null +++ b/python_moonclient/python_moonclient/slaves.py @@ -0,0 +1,61 @@ +import logging +import requests +import copy +from . import config + +logger = logging.getLogger("moonclient.slaves") + + +URL = None +HEADERS = None + + +def init(consul_host, consul_port): + conf_data = config.get_config_data(consul_host, consul_port) + global URL, HEADERS + URL = "http://{}:{}".format( + conf_data['manager_host'], + conf_data['manager_port']) + URL = URL + "{}" + HEADERS = {"content-type": "application/json"} + + +def get_slaves(): + req = requests.get(URL.format("/slaves")) + assert req.status_code == 200 + result = req.json() + assert type(result) is dict + assert "slaves" in result + return result + + +def set_slave(name): + slaves = get_slaves().get("slaves", []) + names = map(lambda x: x['name'], slaves) + assert name in names + req = requests.patch(URL.format("/slaves/{}".format(name)), + headers=HEADERS, + json={ + "op": "replace", + "variable": "configured", + "value": True + }) + assert req.status_code == 200 + result = req.json() + assert type(result) is dict + assert "slaves" in result + return get_slaves() + + +def delete_slave(name): + slaves = get_slaves().get("slaves", []) + names = map(lambda x: x['name'], slaves) + assert name in names + req = requests.patch(URL.format("/slaves/{}".format(name)), + headers=HEADERS, + json={ + "op": "replace", + "variable": "configured", + "value": False + }) + return get_slaves() diff --git a/python_moonclient/setup.py b/python_moonclient/setup.py index 709e3ffa..dcb90365 100644 --- a/python_moonclient/setup.py +++ b/python_moonclient/setup.py @@ -48,7 +48,10 @@ setup( 'moon_delete_pdp = python_moonclient.scripts:delete_pdp', 'moon_delete_policy = python_moonclient.scripts:delete_policy', 'moon_map_pdp_to_project = python_moonclient.scripts:map_pdp_to_project', - 'moon_send_authz_to_wrapper = python_moonclient.scripts:send_authz_to_wrapper' + 'moon_send_authz_to_wrapper = python_moonclient.scripts:send_authz_to_wrapper', + 'moon_get_slaves = python_moonclient.scripts:get_slaves', + 'moon_set_slave = python_moonclient.scripts:set_slave', + 'moon_delete_slave = python_moonclient.scripts:delete_slave' ], } diff --git a/python_moonclient/tests/unit_python/__init__.py b/python_moonclient/tests/unit_python/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/python_moonclient/tests/unit_python/__init__.py diff --git a/python_moondb/python_moondb/api/policy.py b/python_moondb/python_moondb/api/policy.py index 97866bfd..ca313f9a 100644 --- a/python_moondb/python_moondb/api/policy.py +++ b/python_moondb/python_moondb/api/policy.py @@ -22,12 +22,16 @@ class PolicyManager(Managers): policies = self.PolicyManager.get_policies("admin") models = self.ModelManager.get_models("admin") for pdp_key, pdp_value in self.PDPManager.get_pdp(user_id).items(): + if 'security_pipeline' not in pdp_value: + raise exceptions.PdpContentError for policy_id in pdp_value["security_pipeline"]: - if not policies: + if not policies or policy_id not in policies: raise exceptions.PolicyUnknown model_id = policies[policy_id]["model_id"] if not models: raise exceptions.ModelUnknown + if model_id not in models: + raise exceptions.ModelUnknown if meta_rule_id in models[model_id]["meta_rules"]: return policy_id @@ -78,6 +82,8 @@ class PolicyManager(Managers): value.get('name')) perimeter_id = uuid4().hex value.update(k_user['users'][0]) + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown return self.driver.set_subject(policy_id=policy_id, perimeter_id=perimeter_id, value=value) @enforce(("read", "write"), "perimeter") @@ -90,6 +96,8 @@ class PolicyManager(Managers): @enforce(("read", "write"), "perimeter") def add_object(self, user_id, policy_id, perimeter_id=None, value=None): + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown if not perimeter_id: perimeter_id = uuid4().hex return self.driver.set_object(policy_id=policy_id, perimeter_id=perimeter_id, value=value) @@ -104,6 +112,8 @@ class PolicyManager(Managers): @enforce(("read", "write"), "perimeter") def add_action(self, user_id, policy_id, perimeter_id=None, value=None): + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown if not perimeter_id: perimeter_id = uuid4().hex return self.driver.set_action(policy_id=policy_id, perimeter_id=perimeter_id, value=value) @@ -127,6 +137,10 @@ class PolicyManager(Managers): @enforce(("read", "write"), "data") def set_subject_data(self, user_id, policy_id, data_id=None, category_id=None, value=None): + if not category_id: + raise Exception('Invalid category id') + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown if not data_id: data_id = uuid4().hex return self.driver.set_subject_data(policy_id=policy_id, data_id=data_id, category_id=category_id, value=value) @@ -151,6 +165,10 @@ class PolicyManager(Managers): @enforce(("read", "write"), "data") def add_object_data(self, user_id, policy_id, data_id=None, category_id=None, value=None): + if not category_id: + raise Exception('Invalid category id') + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown if not data_id: data_id = uuid4().hex return self.driver.set_object_data(policy_id=policy_id, data_id=data_id, category_id=category_id, value=value) @@ -167,14 +185,18 @@ class PolicyManager(Managers): if not category_id: for cat in available_metadata["action"]: results.append(self.driver.get_action_data(policy_id=policy_id, data_id=data_id, - category_id=cat)) + category_id=cat)) if category_id and category_id in available_metadata["action"]: results.append(self.driver.get_action_data(policy_id=policy_id, data_id=data_id, - category_id=category_id)) + category_id=category_id)) return results @enforce(("read", "write"), "data") def add_action_data(self, user_id, policy_id, data_id=None, category_id=None, value=None): + if not category_id: + raise Exception('Invalid category id') + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown if not data_id: data_id = uuid4().hex return self.driver.set_action_data(policy_id=policy_id, data_id=data_id, category_id=category_id, value=value) @@ -190,6 +212,8 @@ class PolicyManager(Managers): @enforce(("read", "write"), "assignments") def add_subject_assignment(self, user_id, policy_id, subject_id, category_id, data_id): + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown return self.driver.add_subject_assignment(policy_id=policy_id, subject_id=subject_id, category_id=category_id, data_id=data_id) @@ -204,6 +228,8 @@ class PolicyManager(Managers): @enforce(("read", "write"), "assignments") def add_object_assignment(self, user_id, policy_id, object_id, category_id, data_id): + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown return self.driver.add_object_assignment(policy_id=policy_id, object_id=object_id, category_id=category_id, data_id=data_id) @@ -218,6 +244,8 @@ class PolicyManager(Managers): @enforce(("read", "write"), "assignments") def add_action_assignment(self, user_id, policy_id, action_id, category_id, data_id): + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown return self.driver.add_action_assignment(policy_id=policy_id, action_id=action_id, category_id=category_id, data_id=data_id) @@ -232,6 +260,8 @@ class PolicyManager(Managers): @enforce(("read", "write"), "rules") def add_rule(self, user_id, policy_id, meta_rule_id, value): + if not self.get_policies(user_id=user_id, policy_id=policy_id): + raise exceptions.PolicyUnknown return self.driver.add_rule(policy_id=policy_id, meta_rule_id=meta_rule_id, value=value) @enforce(("read", "write"), "rules") diff --git a/python_moondb/tests/unit_python/policies/test_assignments.py b/python_moondb/tests/unit_python/policies/test_assignments.py index ccac205a..707632b0 100755 --- a/python_moondb/tests/unit_python/policies/test_assignments.py +++ b/python_moondb/tests/unit_python/policies/test_assignments.py @@ -1,3 +1,6 @@ +import policies.mock_data as mock_data + + def get_action_assignments(policy_id, action_id=None, category_id=None): from python_moondb.core import PolicyManager return PolicyManager.get_action_assignments("", policy_id, action_id, category_id) @@ -44,7 +47,7 @@ def delete_subject_assignment(policy_id, subject_id, category_id, data_id): def test_get_action_assignments(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() action_id = "action_id_1" category_id = "category_id_1" data_id = "data_id_1" @@ -59,7 +62,7 @@ def test_get_action_assignments(db): def test_get_action_assignments_by_policy_id(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() action_id = "action_id_1" category_id = "category_id_1" data_id = "data_id_1" @@ -77,7 +80,7 @@ def test_get_action_assignments_by_policy_id(db): def test_add_action_assignments(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() action_id = "action_id_1" category_id = "category_id_1" data_id = "data_id_1" @@ -92,9 +95,9 @@ def test_add_action_assignments(db): def test_delete_action_assignment(db): - policy_id = "admin_1" + policy_id = mock_data.get_policy_id() add_action_assignment(policy_id, "", "", "") - policy_id = "admin_2" + policy_id = mock_data.get_policy_id() action_id = "action_id_2" category_id = "category_id_2" data_id = "data_id_2" @@ -112,7 +115,7 @@ def test_delete_action_assignment_with_invalid_policy_id(db): def test_get_object_assignments(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() object_id = "object_id_1" category_id = "category_id_1" data_id = "data_id_1" @@ -127,7 +130,7 @@ def test_get_object_assignments(db): def test_get_object_assignments_by_policy_id(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() object_id_1 = "object_id_1" category_id_1 = "category_id_1" data_id = "data_id_1" @@ -145,7 +148,7 @@ def test_get_object_assignments_by_policy_id(db): def test_add_object_assignments(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() object_id = "object_id_1" category_id = "category_id_1" data_id = "data_id_1" @@ -160,7 +163,7 @@ def test_add_object_assignments(db): def test_delete_object_assignment(db): - policy_id = "admin_1" + policy_id = mock_data.get_policy_id() add_object_assignment(policy_id, "", "", "") object_id = "action_id_2" category_id = "category_id_2" @@ -179,7 +182,7 @@ def test_delete_object_assignment_with_invalid_policy_id(db): def test_get_subject_assignments(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() subject_id = "object_id_1" category_id = "category_id_1" data_id = "data_id_1" @@ -194,7 +197,7 @@ def test_get_subject_assignments(db): def test_get_subject_assignments_by_policy_id(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() subject_id_1 = "subject_id_1" category_id_1 = "category_id_1" data_id = "data_id_1" @@ -212,7 +215,7 @@ def test_get_subject_assignments_by_policy_id(db): def test_add_subject_assignments(db): - policy_id = "admin" + policy_id = mock_data.get_policy_id() subject_id = "subject_id_1" category_id = "category_id_1" data_id = "data_id_1" @@ -227,7 +230,7 @@ def test_add_subject_assignments(db): def test_delete_subject_assignment(db): - policy_id = "admin_1" + policy_id = mock_data.get_policy_id() add_subject_assignment(policy_id, "", "", "") subject_id = "subject_id_2" category_id = "category_id_2" diff --git a/python_moondb/tests/unit_python/policies/test_data.py b/python_moondb/tests/unit_python/policies/test_data.py index 875121eb..67fa44fb 100755 --- a/python_moondb/tests/unit_python/policies/test_data.py +++ b/python_moondb/tests/unit_python/policies/test_data.py @@ -117,8 +117,6 @@ def test_get_action_data(db): def test_get_action_data_with_invalid_category_id(db): policy_id = mock_data.get_policy_id() get_available_metadata(policy_id) - - policy_id = policy_id data_id = "data_id_1" category_id = "action_category_id1" value = { @@ -132,7 +130,7 @@ def test_get_action_data_with_invalid_category_id(db): def test_add_action_data(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() data_id = "data_id_1" category_id = "category_id_1" value = { @@ -145,6 +143,18 @@ def test_add_action_data(db): assert action_data[action_data_id].get('policy_id') == policy_id +def test_add_action_data_with_invalid_category_id(db): + policy_id = mock_data.get_policy_id() + data_id = "data_id_1" + value = { + "name": "action-type", + "description": {"vm-action": "", "storage-action": "", }, + } + with pytest.raises(Exception) as exception_info: + add_action_data(policy_id=policy_id, data_id=data_id, value=value).get('data') + assert str(exception_info.value) == 'Invalid category id' + + def test_delete_action_data(db): policy_id = mock_data.get_policy_id() get_available_metadata(policy_id) @@ -164,8 +174,6 @@ def test_delete_action_data(db): def test_get_object_data(db): policy_id = mock_data.get_policy_id() get_available_metadata(policy_id) - - policy_id = policy_id data_id = "data_id_1" category_id = "object_category_id1" value = { @@ -181,8 +189,6 @@ def test_get_object_data(db): def test_get_object_data_with_invalid_category_id(db): policy_id = mock_data.get_policy_id() get_available_metadata(policy_id) - - policy_id = policy_id data_id = "data_id_1" category_id = "object_category_id1" value = { @@ -196,7 +202,7 @@ def test_get_object_data_with_invalid_category_id(db): def test_add_object_data(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() data_id = "data_id_1" category_id = "object_category_id1" value = { @@ -209,6 +215,18 @@ def test_add_object_data(db): assert object_data[object_data_id].get('policy_id') == policy_id +def test_add_object_data_with_invalid_category_id(db): + policy_id = mock_data.get_policy_id() + data_id = "data_id_1" + value = { + "name": "object-security-level", + "description": {"low": "", "medium": "", "high": ""}, + } + with pytest.raises(Exception) as exception_info: + add_object_data(policy_id=policy_id, data_id=data_id, value=value).get('data') + assert str(exception_info.value) == 'Invalid category id' + + def test_delete_object_data(db): policy_id = mock_data.get_policy_id() get_available_metadata(policy_id) @@ -228,8 +246,6 @@ def test_delete_object_data(db): def test_get_subject_data(db): policy_id = mock_data.get_policy_id() get_available_metadata(policy_id) - - policy_id = policy_id data_id = "data_id_1" category_id = "subject_category_id1" value = { @@ -245,8 +261,6 @@ def test_get_subject_data(db): def test_get_subject_data_with_invalid_category_id(db): policy_id = mock_data.get_policy_id() get_available_metadata(policy_id) - - policy_id = policy_id data_id = "data_id_1" category_id = "subject_category_id1" value = { @@ -260,19 +274,31 @@ def test_get_subject_data_with_invalid_category_id(db): def test_add_subject_data(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() data_id = "data_id_1" category_id = "subject_category_id1" value = { "name": "subject-security-level", "description": {"low": "", "medium": "", "high": ""}, } - subject_data = add_object_data(policy_id, data_id, category_id, value).get('data') + subject_data = add_subject_data(policy_id, data_id, category_id, value).get('data') assert subject_data subject_data_id = list(subject_data.keys())[0] assert subject_data[subject_data_id].get('policy_id') == policy_id +def test_add_subject_data_with_no_category_id(db): + policy_id = mock_data.get_policy_id() + data_id = "data_id_1" + value = { + "name": "subject-security-level", + "description": {"low": "", "medium": "", "high": ""}, + } + with pytest.raises(Exception) as exception_info: + add_subject_data(policy_id=policy_id, data_id=data_id, value=value).get('data') + assert str(exception_info.value) == 'Invalid category id' + + def test_delete_subject_data(db): policy_id = mock_data.get_policy_id() get_available_metadata(policy_id) @@ -290,7 +316,7 @@ def test_delete_subject_data(db): def test_get_actions(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "test_action", "description": "test", @@ -304,7 +330,7 @@ def test_get_actions(db): def test_add_action(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "test_action", "description": "test", @@ -316,7 +342,7 @@ def test_add_action(db): def test_add_action_multiple_times(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "test_action", "description": "test", @@ -330,14 +356,14 @@ def test_add_action_multiple_times(db): "description": "test", "policy_list": ['policy_id_3', 'policy_id_4'] } - action = add_action('policy_id_7', perimeter_id, value) + action = add_action(mock_data.get_policy_id(), perimeter_id, value) assert action action_id = list(action.keys())[0] assert len(action[action_id].get('policy_list')) == 2 def test_delete_action(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "test_action", "description": "test", @@ -358,7 +384,7 @@ def test_delete_action_with_invalid_perimeter_id(db): def test_get_objects(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "test_object", "description": "test", @@ -372,7 +398,7 @@ def test_get_objects(db): def test_add_object(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "test_object", "description": "test", @@ -384,7 +410,7 @@ def test_add_object(db): def test_add_objects_multiple_times(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "test_object", "description": "test", @@ -398,14 +424,14 @@ def test_add_objects_multiple_times(db): "description": "test", "policy_list": ['policy_id_3', 'policy_id_4'] } - added_object = add_object('policy_id_7', perimeter_id, value) + added_object = add_object(mock_data.get_policy_id(), perimeter_id, value) assert added_object object_id = list(added_object.keys())[0] assert len(added_object[object_id].get('policy_list')) == 2 def test_delete_object(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "test_object", "description": "test", @@ -426,7 +452,7 @@ def test_delete_object_with_invalid_perimeter_id(db): def test_get_subjects(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "testuser", "description": "test", @@ -440,7 +466,7 @@ def test_get_subjects(db): def test_add_subject(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "testuser", "description": "test", @@ -452,7 +478,7 @@ def test_add_subject(db): def test_add_subjects_multiple_times(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "testuser", "description": "test", @@ -466,14 +492,14 @@ def test_add_subjects_multiple_times(db): "description": "test", "policy_list": ['policy_id_3', 'policy_id_4'] } - subject = add_subject('policy_id_7', perimeter_id, value) + subject = add_subject(mock_data.get_policy_id(), perimeter_id, value) assert subject subject_id = list(subject.keys())[0] assert len(subject[subject_id].get('policy_list')) == 2 def test_delete_subject(db): - policy_id = "policy_id_1" + policy_id = mock_data.get_policy_id() value = { "name": "testuser", "description": "test", diff --git a/python_moondb/tests/unit_python/policies/test_policies.py b/python_moondb/tests/unit_python/policies/test_policies.py index 487cb6a1..148034ef 100755 --- a/python_moondb/tests/unit_python/policies/test_policies.py +++ b/python_moondb/tests/unit_python/policies/test_policies.py @@ -231,7 +231,7 @@ def test_get_rules(db): "instructions": ({"decision": "grant"}), "enabled": "", } - policy_id = "1" + policy_id = mock_data.get_policy_id() meta_rule_id = "1" add_rule(policy_id, meta_rule_id, value) value = { @@ -239,7 +239,6 @@ def test_get_rules(db): "instructions": ({"decision": "grant"}), "enabled": "", } - policy_id = "1" meta_rule_id = "1" add_rule(policy_id, meta_rule_id, value) rules = get_rules(policy_id, meta_rule_id) @@ -261,7 +260,7 @@ def test_add_rule(db): "instructions": ({"decision": "grant"}), "enabled": "", } - policy_id = "1" + policy_id = mock_data.get_policy_id() meta_rule_id = "1" rules = add_rule(policy_id, meta_rule_id, value) assert rules @@ -279,7 +278,7 @@ def test_delete_rule(db): "instructions": ({"decision": "grant"}), "enabled": "", } - policy_id = "2" + policy_id = mock_data.get_policy_id() meta_rule_id = "2" rules = add_rule(policy_id, meta_rule_id, value) rule_id = list(rules.keys())[0] diff --git a/python_moonutilities/Changelog b/python_moonutilities/Changelog index 2c4c02a8..ffc03809 100644 --- a/python_moonutilities/Changelog +++ b/python_moonutilities/Changelog @@ -78,3 +78,7 @@ CHANGES 1.4.5 ----- - Add PdpKeystoneMappingConflict exception + +1.4.6 +----- +- Add WrapperConflict, PipelineConflict, SlaveNameUnknown exceptions diff --git a/python_moonutilities/python_moonutilities/__init__.py b/python_moonutilities/python_moonutilities/__init__.py index bcd7e545..741ba4f6 100644 --- a/python_moonutilities/python_moonutilities/__init__.py +++ b/python_moonutilities/python_moonutilities/__init__.py @@ -3,6 +3,6 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -__version__ = "1.4.5" +__version__ = "1.4.6" diff --git a/python_moonutilities/python_moonutilities/cache.py b/python_moonutilities/python_moonutilities/cache.py index 1ea59d3a..1bb9d09e 100644 --- a/python_moonutilities/python_moonutilities/cache.py +++ b/python_moonutilities/python_moonutilities/cache.py @@ -101,14 +101,14 @@ class Cache(object): raise exceptions.PolicyUnknown("Cannot find policy within policy_id {}".format(policy_id)) if policy_id in self.subjects: - for _subject_id, _subject_dict in self.__SUBJECTS[policy_id].items(): + for _subject_id, _subject_dict in self.subjects[policy_id].items(): if "name" in _subject_dict and _subject_dict["name"] == name: return _subject_id self.__update_subjects(policy_id) if policy_id in self.subjects: - for _subject_id, _subject_dict in self.__SUBJECTS[policy_id].items(): + for _subject_id, _subject_dict in self.subjects[policy_id].items(): if "name" in _subject_dict and _subject_dict["name"] == name: return _subject_id @@ -488,6 +488,20 @@ class Cache(object): logger.warning("Cannot find 'security_pipeline' " "key within pdp ") + def get_meta_rule_ids_from_pdp_value(self, pdp_value): + meta_rules = [] + if "security_pipeline" in pdp_value: + for policy_id in pdp_value["security_pipeline"]: + if policy_id not in self.policies or "model_id" not in self.policies[policy_id]: + raise exceptions.PolicyUnknown("Cannot find 'models' key") + model_id = self.policies[policy_id]["model_id"] + if model_id not in self.models or 'meta_rules' not in self.models[model_id]: + raise exceptions.ModelNotFound("Cannot find 'models' key") + for meta_rule in self.models[model_id]["meta_rules"]: + meta_rules.append(meta_rule) + return meta_rules + raise exceptions.PdpContentError + def get_pdp_from_keystone_project(self, keystone_project_id): for pdp_key, pdp_value in self.pdp.items(): if "keystone_project_id" in pdp_value and \ @@ -566,8 +580,8 @@ class Cache(object): :return: """ if all(k in container_data for k in ("keystone_project_id", "name", "container_id", "policy_id", - "meta_rule_id", "port")) \ - and all(k in container_data['port'] for k in ("PublicPort", "Type", "IP", "PrivatePort")): + "meta_rule_id", "port")) \ + and all(k in container_data['port'] for k in ("PublicPort", "Type", "IP", "PrivatePort")): self.__CONTAINERS[uuid4().hex] = { "keystone_project_id": container_data['keystone_project_id'], @@ -641,7 +655,7 @@ class Cache(object): container_ids = [] for pdp_id, pdp_value, in self.__PDP.items(): if pdp_value: - if all(k in pdp_value for k in ("keystone_project_id", "security_pipeline")) \ + if all(k in pdp_value for k in ("keystone_project_id", "security_pipeline")) \ and pdp_value["keystone_project_id"] == keystone_project_id: for policy_id in pdp_value["security_pipeline"]: if policy_id in self.policies and "model_id" in self.policies[policy_id]: @@ -677,4 +691,3 @@ class Cache(object): "and may not contains 'model_id' key".format(policy_id)) self.__CONTAINER_CHAINING[keystone_project_id] = container_ids - diff --git a/python_moonutilities/python_moonutilities/context.py b/python_moonutilities/python_moonutilities/context.py index 626b25dc..1d25cda2 100644 --- a/python_moonutilities/python_moonutilities/context.py +++ b/python_moonutilities/python_moonutilities/context.py @@ -14,39 +14,35 @@ logger = logging.getLogger("moon.utilities." + __name__) class Context: def __init__(self, init_context, cache): + if init_context is None: + raise Exception("Invalid context content object") + self.cache = cache self.__keystone_project_id = init_context.get("project_id") - self.__pdp_id = None - self.__pdp_value = None - for _pdp_key, _pdp_value in self.cache.pdp.items(): - if _pdp_value["keystone_project_id"] == self.__keystone_project_id: - self.__pdp_id = _pdp_key - self.__pdp_value = copy.deepcopy(_pdp_value) - break - if not self.__pdp_value: + self.__pdp_id = self.cache.get_pdp_from_keystone_project(self.__keystone_project_id) + + if not self.__pdp_id: raise exceptions.AuthzException( "Cannot create context for authz " "with Keystone project ID {}".format( self.__keystone_project_id - )) + )) + self.__pdp_value = copy.deepcopy(self.cache.pdp[self.__pdp_id]) + self.__subject = init_context.get("subject_name") self.__object = init_context.get("object_name") self.__action = init_context.get("action_name") - self.__current_request = None self.__request_id = init_context.get("req_id") self.__cookie = init_context.get("cookie") self.__manager_url = init_context.get("manager_url") self.__interface_name = init_context.get("interface_name") + self.__current_request = None + self.__index = -1 # self.__init_initial_request() - self.__headers = [] - policies = self.cache.policies - models = self.cache.models - for policy_id in self.__pdp_value["security_pipeline"]: - model_id = policies[policy_id]["model_id"] - for meta_rule in models[model_id]["meta_rules"]: - self.__headers.append(meta_rule) + self.__meta_rule_ids = self.cache.get_meta_rule_ids_from_pdp_value(self.__pdp_value) self.__meta_rules = self.cache.meta_rules + self.__pdp_set = {} # self.__init_pdp_set() @@ -63,20 +59,25 @@ class Context: @property def current_state(self): - return self.__pdp_set[self.__headers[self.__index]]['effect'] + self.__validate_meta_rule_content(self.__meta_rule_ids[self.__index]) + return self.__pdp_set[self.__meta_rule_ids[self.__index]]['effect'] @current_state.setter def current_state(self, state): if state not in ("grant", "deny", "passed"): state = "passed" - self.__pdp_set[self.__headers[self.__index]]['effect'] = state + self.__validate_meta_rule_content(self.__meta_rule_ids[self.__index]) + self.__pdp_set[self.__meta_rule_ids[self.__index]]['effect'] = state @current_state.deleter def current_state(self): - self.__pdp_set[self.__headers[self.__index]]['effect'] = "unset" + self.__validate_meta_rule_content(self.__meta_rule_ids[self.__index]) + self.__pdp_set[self.__meta_rule_ids[self.__index]]['effect'] = "unset" @property def current_policy_id(self): + if "security_pipeline" not in self.__pdp_value: + raise exceptions.AuthzException('Cannot find security_pipeline key within pdp.') return self.__pdp_value["security_pipeline"][self.__index] @current_policy_id.setter @@ -88,6 +89,8 @@ class Context: pass def __init_current_request(self): + if "security_pipeline" not in self.__pdp_value: + raise exceptions.PdpContentError self.__subject = self.cache.get_subject( self.__pdp_value["security_pipeline"][self.__index], self.__subject) @@ -100,11 +103,11 @@ class Context: self.__current_request = dict(self.initial_request) def __init_pdp_set(self): - for header in self.__headers: - self.__pdp_set[header] = dict() - self.__pdp_set[header]["meta_rules"] = self.__meta_rules[header] - self.__pdp_set[header]["target"] = self.__add_target(header) - self.__pdp_set[header]["effect"] = "unset" + for meta_rule_id in self.__meta_rule_ids: + self.__pdp_set[meta_rule_id] = dict() + self.__pdp_set[meta_rule_id]["meta_rules"] = self.__meta_rules[meta_rule_id] + self.__pdp_set[meta_rule_id]["target"] = self.__add_target(meta_rule_id) + self.__pdp_set[meta_rule_id]["effect"] = "unset" self.__pdp_set["effect"] = "deny" # def update_target(self, context): @@ -151,23 +154,37 @@ class Context: _subject = self.__current_request["subject"] _object = self.__current_request["object"] _action = self.__current_request["action"] + meta_rules = self.cache.meta_rules policy_id = self.cache.get_policy_from_meta_rules(meta_rule_id) + + if 'subject_categories' not in meta_rules[meta_rule_id]: + raise exceptions.MetaRuleContentError(" 'subject_categories' key not found ") + for sub_cat in meta_rules[meta_rule_id]['subject_categories']: if sub_cat not in result: result[sub_cat] = [] result[sub_cat].extend( self.cache.get_subject_assignments(policy_id, _subject, sub_cat)) + + if 'object_categories' not in meta_rules[meta_rule_id]: + raise exceptions.MetaRuleContentError(" 'object_categories' key not found ") + for obj_cat in meta_rules[meta_rule_id]['object_categories']: if obj_cat not in result: result[obj_cat] = [] result[obj_cat].extend( self.cache.get_object_assignments(policy_id, _object, obj_cat)) + + if 'action_categories' not in meta_rules[meta_rule_id]: + raise exceptions.MetaRuleContentError(" 'action_categories' key not found ") + for act_cat in meta_rules[meta_rule_id]['action_categories']: if act_cat not in result: result[act_cat] = [] result[act_cat].extend( self.cache.get_action_assignments(policy_id, _action, act_cat)) + return result def __repr__(self): @@ -181,7 +198,7 @@ pdp_set: {pdp_set} id=self.__pdp_id, current_request=self.__current_request, request_id=self.__request_id, - headers=self.__headers, + headers=self.__meta_rule_ids, pdp_set=self.__pdp_set, index=self.__index ) @@ -190,7 +207,7 @@ pdp_set: {pdp_set} return { "initial_request": copy.deepcopy(self.initial_request), "current_request": copy.deepcopy(self.__current_request), - "headers": copy.deepcopy(self.__headers), + "headers": copy.deepcopy(self.__meta_rule_ids), "index": copy.deepcopy(self.__index), "pdp_set": copy.deepcopy(self.__pdp_set), "request_id": copy.deepcopy(self.__request_id), @@ -265,11 +282,12 @@ pdp_set: {pdp_set} @property def current_request(self): if not self.__current_request: - self.__current_request = copy.deepcopy(self.initial_request) + self.__current_request = dict(self.initial_request) return self.__current_request @current_request.setter def current_request(self, value): + self.__current_request = copy.deepcopy(value) # Note (asteroide): if the current request is modified, # we must update the PDP Set. @@ -280,17 +298,22 @@ pdp_set: {pdp_set} self.__current_request = {} self.__pdp_set = {} + ''' + [Note ] Refactor name of headers to meta_rule_ids done , + may need to refactor getter and setter of headers + ''' + @property def headers(self): - return self.__headers + return self.__meta_rule_ids @headers.setter - def headers(self, headers): - self.__headers = headers + def headers(self, meta_rule_ids): + self.__meta_rule_ids = meta_rule_ids @headers.deleter def headers(self): - self.__headers = list() + self.__meta_rule_ids = list() @property def index(self): @@ -316,4 +339,6 @@ pdp_set: {pdp_set} def pdp_set(self): self.__pdp_set = {} - + def __validate_meta_rule_content(self, meta_rules): + if 'effect' not in meta_rules: + raise exceptions.PdpContentError diff --git a/python_moonutilities/python_moonutilities/exceptions.py b/python_moonutilities/python_moonutilities/exceptions.py index 6db7bf01..f3763428 100644 --- a/python_moonutilities/python_moonutilities/exceptions.py +++ b/python_moonutilities/python_moonutilities/exceptions.py @@ -444,7 +444,7 @@ class MetaRuleExisting(AdminMetaRule): class MetaRuleContentError(AdminMetaRule): - description = _("Invalid content of pdp.") + description = _("Invalid content of meta rule.") code = 400 title = 'Meta Rule Error' logger = "ERROR" @@ -535,6 +535,41 @@ class ContainerMissing(DockerError): logger = "ERROR" +class WrapperConflict(MoonError): + description = _("A Wrapper already exist for the specified slave.") + code = 409 + title = 'Wrapper conflict' + logger = "ERROR" + + +class PipelineConflict(MoonError): + description = _("A Pipeline already exist for the specified slave.") + code = 409 + title = 'Pipeline conflict' + logger = "ERROR" + + +class PipelineUnknown(MoonError): + description = _("This Pipeline is unknown from the system.") + code = 400 + title = 'Pipeline Unknown' + logger = "ERROR" + + +class WrapperUnknown(MoonError): + description = _("This Wrapper is unknown from the system.") + code = 400 + title = 'Wrapper Unknown' + logger = "ERROR" + + +class SlaveNameUnknown(MoonError): + description = _("The slave is unknown.") + code = 400 + title = 'Slave Unknown' + logger = "Error" + + class PdpUnknown(MoonError): description = _("The pdp is unknown.") code = 400 diff --git a/tests/functional/run_tests.sh b/tests/functional/run_tests.sh index ced0f9f7..c5cbabbb 100644 --- a/tests/functional/run_tests.sh +++ b/tests/functional/run_tests.sh @@ -1,3 +1,13 @@ #!/usr/bin/env bash echo "starting Moon Functional Tests" + +COMPONENTS="moon_authz, moon_interface, moon_manager, moon_orchestrator, moon_wrapper" + +for dir in ${COMPONENTS}; do + echo "Testing component ${dir}" + cd ${MOON_HOME}/${dir} + docker run --rm --volume $(pwd):/data wukongsun/moon_forming:latest /bin/bash /root/switch.sh functest +done + +# TODO: download tests results diff --git a/tests/functional/run_tests_for_component.sh b/tests/functional/run_tests_for_component.sh new file mode 100644 index 00000000..fd9ab7fa --- /dev/null +++ b/tests/functional/run_tests_for_component.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CUR_PWD=$(pwd) +INPUT_FILE=../tools/moon_kubernetes/templates/moon_forming_functest.yaml +OUTPUT_FILE=tests/functional_pod/moon_forming_functest.yaml + +echo current working directory: ${CUR_PWD} + +cat ${INPUT_FILE} | sed "s|{{PATH}}|${CUR_PWD}|" > ${OUTPUT_FILE} + +kubectl create -f ${OUTPUT_FILE} + +sleep 5 +kubectl get -n moon jobs +echo OUTPUT is $? +if [ "$?" -ne 0 ] +then + sleep 5 + kubectl get -n moon jobs +fi + +echo "waiting for FuncTests (it may takes time)..." +echo -e "\033[35m" +sed '/<END OF JOB>/q' <(kubectl logs -n moon jobs/functest -f) +echo -e "\033[m" + +kubectl delete -f ${OUTPUT_FILE} diff --git a/tools/moon_jenkins/Dockerfile b/tools/moon_jenkins/Dockerfile index 573d4ac8..058f388c 100644 --- a/tools/moon_jenkins/Dockerfile +++ b/tools/moon_jenkins/Dockerfile @@ -3,7 +3,6 @@ FROM jenkinsci/blueocean ENV JAVA_OPTS="-Djenkins.install.runSetupWizard=false" COPY security.groovy /usr/share/jenkins/ref/init.groovy.d/security.groovy -COPY setenv.groovy /usr/share/jenkins/ref/init.groovy.d/setenv.groovy COPY plugins.txt /usr/share/jenkins/ref/plugins.txt RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt
\ No newline at end of file diff --git a/tools/moon_jenkins/Jenkinsfile b/tools/moon_jenkins/Jenkinsfile deleted file mode 100644 index 7e0e07c0..00000000 --- a/tools/moon_jenkins/Jenkinsfile +++ /dev/null @@ -1,24 +0,0 @@ -pipeline { - agent { - docker { - image 'wukongsun/moon_python_unit_test' - args '-e moon_home=${moon_home}' - } - } - stages { - stage('Python Unit Test') { - steps { - script { - sh("cd ${moon_home}/tests/python_unit") - sh("bash run_tests") - } - } - } - stage('Functional Test') { - script { - sh("cd ${moon_home}/tests/functional") - sh("bash run_tests") - } - } - } -}
\ No newline at end of file diff --git a/tools/moon_jenkins/docker-compose.yml b/tools/moon_jenkins/docker-compose.yml index 2329f7bd..eb9354ce 100644 --- a/tools/moon_jenkins/docker-compose.yml +++ b/tools/moon_jenkins/docker-compose.yml @@ -1,12 +1,10 @@ version: '3.1' services: - jenkins: build: context: . - args: - image: blueocean:v0.3 + image: blueocean:v0.4 ports: - 8080:8080 - 50000:50000 @@ -19,4 +17,4 @@ services: user: root volumes: - jenkins-data: + jenkins-data:
\ No newline at end of file diff --git a/tools/moon_jenkins/plugins.txt b/tools/moon_jenkins/plugins.txt index 2463d029..65bae872 100644 --- a/tools/moon_jenkins/plugins.txt +++ b/tools/moon_jenkins/plugins.txt @@ -97,4 +97,4 @@ ssh-slaves pam-auth ldap email-ext -locale +locale
\ No newline at end of file diff --git a/tools/moon_jenkins/setenv.groovy b/tools/moon_jenkins/setenv.groovy deleted file mode 100644 index ab2dc137..00000000 --- a/tools/moon_jenkins/setenv.groovy +++ /dev/null @@ -1,34 +0,0 @@ -#!groovy - -import jenkins.* -import jenkins.model.* -import hudson.* -import hudson.model.* - -instance = Jenkins.getInstance() -globalNodeProperties = instance.getGlobalNodeProperties() - -envVarsNodePropertyList = globalNodeProperties.getAll(hudson.slaves.EnvironmentVariablesNodeProperty.class) - -newEnvVarsNodeProperty = null -envVars = null - -if (envVarsNodePropertyList == null || envVarsNodePropertyList.size() == 0) { - newEnvVarsNodeProperty = new hudson.slaves.EnvironmentVariablesNodeProperty(); - globalNodeProperties.add(newEnvVarsNodeProperty) - envVars = newEnvVarsNodeProperty.getEnvVars() -} else { - envVars = envVarsNodePropertyList.get(0).getEnvVars() -} - -http_proxy = System.getenv()['http_proxy'] -https_proxy = System.getenv()['https_proxy'] - -if (http_proxy) { - envVars.put("http_proxy", System.getenv()['http_proxy']) -} -if (https_proxy) { - envVars.put("https_proxy", System.getenv()['https_proxy']) -} - -instance.save() diff --git a/tools/moon_kubernetes/templates/moon_forming_functest.yaml b/tools/moon_kubernetes/templates/moon_forming_functest.yaml new file mode 100644 index 00000000..4cb2c3a0 --- /dev/null +++ b/tools/moon_kubernetes/templates/moon_forming_functest.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: functest + namespace: moon +spec: + template: + metadata: + name: functest + spec: + containers: + - name: functest + image: wukongsun/moon_forming:dev + env: + - name: COMMAND + value: "functest" + volumeMounts: + - name: config-volume + mountPath: /etc/moon + - name: tests-volume + mountPath: /data + volumes: + - name: config-volume + configMap: + name: moon-config + - name: tests-volume + hostPath: + path: "{{PATH}}" + restartPolicy: Never + #backoffLimit: 4
\ No newline at end of file diff --git a/tools/moon_kubernetes/templates/moon_gui.yaml b/tools/moon_kubernetes/templates/moon_gui.yaml index 2d355216..7c6ab732 100644 --- a/tools/moon_kubernetes/templates/moon_gui.yaml +++ b/tools/moon_kubernetes/templates/moon_gui.yaml @@ -13,7 +13,7 @@ spec: hostname: gui containers: - name: gui - image: wukongsun/moon_gui:v4.3.1 + image: wukongsun/moon_gui:latest env: - name: MANAGER_HOST value: "127.0.0.1" |