diff options
18 files changed, 434 insertions, 94 deletions
diff --git a/moon_authz/moon_authz/api/authorization.py b/moon_authz/moon_authz/api/authorization.py index c83dd72c..ea177d81 100644 --- a/moon_authz/moon_authz/api/authorization.py +++ b/moon_authz/moon_authz/api/authorization.py @@ -73,7 +73,7 @@ class Authz(Resource): return response def run(self): - logger.info("self.context.pdp_set={}".format(self.context.pdp_set)) + logger.debug("self.context.pdp_set={}".format(self.context.pdp_set)) result, message = self.__check_rules() if result: return self.__exec_instructions(result) diff --git a/moon_authz/moon_authz/server.py b/moon_authz/moon_authz/server.py index 8715bd87..0cc5f6fc 100644 --- a/moon_authz/moon_authz/server.py +++ b/moon_authz/moon_authz/server.py @@ -6,7 +6,7 @@ import os import logging from moon_authz.http_server import HTTPServer as Server -from python_moonutilities import configuration +from python_moonutilities import configuration, exceptions logger = logging.getLogger("moon.authz.server") @@ -21,12 +21,15 @@ def create_server(): meta_rule_id = os.getenv("META_RULE_ID") keystone_project_id = os.getenv("KEYSTONE_PROJECT_ID") logger.info("component_type={}".format(component_type)) - conf = configuration.get_configuration("plugins/{}".format(component_type)) - conf["plugins/{}".format(component_type)]['id'] = component_id - hostname = conf["plugins/{}".format(component_type)].get('hostname', - component_id) - port = conf["plugins/{}".format(component_type)].get('port', tcp_port) - bind = conf["plugins/{}".format(component_type)].get('bind', "0.0.0.0") + conf = configuration.get_plugins() + # conf = configuration.get_configuration("plugins/{}".format(component_type)) + # conf["plugins/{}".format(component_type)]['id'] = component_id + if component_type not in conf: + raise exceptions.ConsulComponentNotFound("{} not found".format( + component_type)) + hostname = conf[component_type].get('hostname', component_id) + port = conf[component_type].get('port', tcp_port) + bind = conf[component_type].get('bind', "0.0.0.0") logger.info("Starting server with IP {} on port {} bind to {}".format( hostname, port, bind)) diff --git a/moon_forming/conf2consul.py b/moon_forming/conf2consul.py index 46c99d5c..148bf923 100644 --- a/moon_forming/conf2consul.py +++ b/moon_forming/conf2consul.py @@ -86,14 +86,13 @@ def main(): # put("slave", data_config["slave"]) # put("docker", data_config["docker"]) put("logging", data_config["logging"]) - put("components_port_start", data_config["components"]["port_start"]) + # put("components_port_start", data_config["components"]["port_start"]) for _key, _value in data_config["components"].items(): - if type(_value) is dict: - put("components/{}".format(_key), data_config["components"][_key]) + put("components/{}".format(_key), data_config["components"][_key]) - for _key, _value in data_config["plugins"].items(): - put("plugins/{}".format(_key), data_config["plugins"][_key]) + # for _key, _value in data_config["plugins"].items(): + # put("plugins/{}".format(_key), data_config["plugins"][_key]) for _key, _value in data_config["openstack"].items(): put("openstack/{}".format(_key), data_config["openstack"][_key]) diff --git a/moon_interface/moon_interface/server.py b/moon_interface/moon_interface/server.py index 13955c3e..0af1fd06 100644 --- a/moon_interface/moon_interface/server.py +++ b/moon_interface/moon_interface/server.py @@ -13,15 +13,16 @@ logger = logging.getLogger("moon.interface.server") def create_server(): configuration.init_logging() try: - conf = configuration.get_configuration("components/interface") - hostname = conf["components/interface"].get("hostname", "interface") - port = conf["components/interface"].get("port", 80) - bind = conf["components/interface"].get("bind", "127.0.0.1") + conf = configuration.get_configuration("components/pipeline").get( + "components/pipeline", {}).get("interface", {}) + hostname = conf.get("hostname", "pipeline") + port = conf.get("port", 80) + bind = conf.get("bind", "127.0.0.1") except exceptions.ConsulComponentNotFound: hostname = "interface" bind = "127.0.0.1" port = 80 - configuration.add_component(uuid="interface", + configuration.add_component(uuid="pipeline", name=hostname, port=port, bind=bind) diff --git a/moon_manager/tests/unit_python/api/meta_rules_test.py b/moon_manager/tests/unit_python/api/meta_rules_test.py new file mode 100644 index 00000000..b5b1ecf8 --- /dev/null +++ b/moon_manager/tests/unit_python/api/meta_rules_test.py @@ -0,0 +1,69 @@ +import json +import api.utilities as utilities + + +def get_meta_rules(client): + req = client.get("/meta_rules") + meta_rules = utilities.get_json(req.data) + return req, meta_rules + + +def add_meta_rules(client, name): + data = { + "name": name, + "subject_categories": ["subject_category_id1", + "subject_category_id2"], + "object_categories": ["object_category_id1"], + "action_categories": ["action_category_id1"] + } + req = client.post("/meta_rules", data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + meta_rules = utilities.get_json(req.data) + return req, meta_rules + + +def delete_meta_rules(client, name): + request, meta_rules = get_meta_rules(client) + for key, value in meta_rules['meta_rules'].items(): + if value['name'] == name: + req = client.delete("/meta_rules/{}".format(key)) + break + return req + + +def delete_meta_rules_without_id(client): + req = client.delete("/meta_rules/{}".format("")) + return req + + +def test_get_meta_rules(): + client = utilities.register_client() + req, meta_rules = get_meta_rules(client) + assert req.status_code == 200 + assert isinstance(meta_rules, dict) + assert "meta_rules" in meta_rules + + +def test_add_meta_rules(): + client = utilities.register_client() + req, meta_rules = add_meta_rules(client, "testuser") + assert req.status_code == 200 + assert isinstance(meta_rules, dict) + value = list(meta_rules["meta_rules"].values())[0] + assert "meta_rules" in meta_rules + assert value['name'] == "testuser" + assert value["subject_categories"][0] == "subject_category_id1" + assert value["object_categories"][0] == "object_category_id1" + assert value["action_categories"][0] == "action_category_id1" + + +def test_delete_meta_rules(): + client = utilities.register_client() + req = delete_meta_rules(client, "testuser") + assert req.status_code == 200 + + +def test_delete_meta_rules_without_id(): + client = utilities.register_client() + req = delete_meta_rules_without_id(client) + assert req.status_code == 500 diff --git a/moon_manager/tests/unit_python/api/test_models.py b/moon_manager/tests/unit_python/api/test_models.py new file mode 100644 index 00000000..3c205d1d --- /dev/null +++ b/moon_manager/tests/unit_python/api/test_models.py @@ -0,0 +1,67 @@ +import json +import api.utilities as utilities + + +def get_models(client): + req = client.get("/models") + models = utilities.get_json(req.data) + return req, models + + +def add_models(client, name): + data = { + "name": name, + "description": "description of {}".format(name), + "meta_rules": ["meta_rule_id1", "meta_rule_id2"] + } + req = client.post("/models", data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + models = utilities.get_json(req.data) + return req, models + + +def delete_models(client, name): + request, models = get_models(client) + for key, value in models['models'].items(): + if value['name'] == name: + req = client.delete("/models/{}".format(key)) + break + return req + + +def delete_models_without_id(client): + req = client.delete("/models/{}".format("")) + return req + + +def test_get_models(): + client = utilities.register_client() + req, models= get_models(client) + assert req.status_code == 200 + assert isinstance(models, dict) + assert "models" in models + + +def test_add_models(): + client = utilities.register_client() + req, models = add_models(client, "testuser") + assert req.status_code == 200 + assert isinstance(models, dict) + value = list(models["models"].values())[0] + assert "models" in models + assert value['name'] == "testuser" + assert value["description"] == "description of {}".format("testuser") + assert value["meta_rules"][0] == "meta_rule_id1" + + +def test_delete_models(): + client = utilities.register_client() + req = delete_models(client, "testuser") + assert req.status_code == 200 + + +def test_delete_models_without_id(): + client = utilities.register_client() + req = delete_models_without_id(client) + assert req.status_code == 500 + diff --git a/moon_manager/tests/unit_python/api/test_policies.py b/moon_manager/tests/unit_python/api/test_policies.py new file mode 100644 index 00000000..4d4e387e --- /dev/null +++ b/moon_manager/tests/unit_python/api/test_policies.py @@ -0,0 +1,69 @@ +import json +import api.utilities as utilities + + +def get_policies(client): + req = client.get("/policies") + policies = utilities.get_json(req.data) + return req, policies + + +def add_policies(client, name): + data = { + "name": name, + "description": "description of {}".format(name), + "model_id": "modelId", + "genre": "genre" + } + req = client.post("/policies", data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + policies = utilities.get_json(req.data) + return req, policies + + +def delete_policies(client, name): + request, policies = get_policies(client) + for key, value in policies['policies'].items(): + if value['name'] == name: + req = client.delete("/policies/{}".format(key)) + break + return req + + +def delete_policies_without_id(client): + req = client.delete("/policies/{}".format("")) + return req + + +def test_get_policies(): + client = utilities.register_client() + req, policies = get_policies(client) + assert req.status_code == 200 + assert isinstance(policies, dict) + assert "policies" in policies + + +def test_add_policies(): + client = utilities.register_client() + req, policies = add_policies(client, "testuser") + assert req.status_code == 200 + assert isinstance(policies, dict) + value = list(policies["policies"].values())[0] + assert "policies" in policies + assert value['name'] == "testuser" + assert value["description"] == "description of {}".format("testuser") + assert value["model_id"] == "modelId" + assert value["genre"] == "genre" + + +def test_delete_policies(): + client = utilities.register_client() + req = delete_policies(client, "testuser") + assert req.status_code == 200 + + +def test_delete_policies_without_id(): + client = utilities.register_client() + req = delete_policies_without_id(client) + assert req.status_code == 500 + diff --git a/moon_orchestrator/moon_orchestrator/http_server.py b/moon_orchestrator/moon_orchestrator/http_server.py index 7105ea7a..00be0335 100644 --- a/moon_orchestrator/moon_orchestrator/http_server.py +++ b/moon_orchestrator/moon_orchestrator/http_server.py @@ -179,7 +179,6 @@ class HTTPServer(Server): logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) - # TODO: get data from consul data = [{ "name": hostname + "-" + get_random_name(), "container": container, @@ -217,15 +216,15 @@ class HTTPServer(Server): return plugins = configuration.get_plugins() - conf = configuration.get_configuration("components/interface") - i_hostname = conf["components/interface"].get("hostname", "interface") - i_port = conf["components/interface"].get("port", 80) - i_container = conf["components/interface"].get( + conf = configuration.get_configuration("components/pipeline") + # i_hostname = conf["components/pipeline"].get("interface").get("hostname", "interface") + i_port = conf["components/pipeline"].get("interface").get("port", 80) + i_container = conf["components/pipeline"].get("interface").get( "container", "wukongsun/moon_interface:v4.3") data = [ { - "name": i_hostname + "-" + get_random_name(), + "name": "pipeline-" + get_random_name(), "container": i_container, "port": i_port, 'pdp_id': pdp_id, diff --git a/moon_orchestrator/tests/unit_python/utilities.py b/moon_orchestrator/tests/unit_python/utilities.py index aec03d9d..d64e4c7b 100644 --- a/moon_orchestrator/tests/unit_python/utilities.py +++ b/moon_orchestrator/tests/unit_python/utilities.py @@ -144,7 +144,7 @@ COMPONENTS = ( "slave", "components/manager", "components/orchestrator", - "components/interface", + "components/pipeline", "components/wrapper", ) diff --git a/moon_wrapper/moon_wrapper/api/generic.py b/moon_wrapper/moon_wrapper/api/generic.py index 7dd44fb4..498513c7 100644 --- a/moon_wrapper/moon_wrapper/api/generic.py +++ b/moon_wrapper/moon_wrapper/api/generic.py @@ -7,13 +7,13 @@ Those API are helping API used to manage the Moon platform. """ from flask_restful import Resource, request -from oslo_log import log as logging +import logging import moon_wrapper.api from python_moonutilities.security_functions import check_auth __version__ = "0.1.0" -LOG = logging.getLogger("moon.manager.api." + __name__) +logger = logging.getLogger("moon.manager.api." + __name__) class Status(Resource): @@ -125,7 +125,7 @@ class API(Resource): if endpoint_id in api_desc[group_id]: return {group_id: {endpoint_id: api_desc[group_id][endpoint_id]}} elif len(endpoint_id) > 0: - LOG.error("Unknown endpoint_id {}".format(endpoint_id)) + logger.error("Unknown endpoint_id {}".format(endpoint_id)) return {"error": "Unknown endpoint_id {}".format(endpoint_id)} return {group_id: api_desc[group_id]} return api_desc diff --git a/moon_wrapper/moon_wrapper/api/oslowrapper.py b/moon_wrapper/moon_wrapper/api/oslowrapper.py index a422ee42..03bdfc69 100644 --- a/moon_wrapper/moon_wrapper/api/oslowrapper.py +++ b/moon_wrapper/moon_wrapper/api/oslowrapper.py @@ -16,7 +16,7 @@ from python_moonutilities import exceptions __version__ = "0.1.0" -LOG = logging.getLogger("moon.wrapper.api." + __name__) +logger = logging.getLogger("moon.wrapper.api." + __name__) class OsloWrapper(Resource): @@ -35,7 +35,7 @@ class OsloWrapper(Resource): self.TIMEOUT = 5 def post(self): - LOG.debug("POST {}".format(request.form)) + logger.debug("POST {}".format(request.form)) response = flask.make_response("False") if self.manage_data(): response = flask.make_response("True") @@ -62,16 +62,16 @@ class OsloWrapper(Resource): @staticmethod def __get_project_id(target, credentials): - LOG.info("__get_project_id {}".format(target)) + logger.info("__get_project_id {}".format(target)) return target.get("project_id", "none") def get_interface_url(self, project_id): - LOG.info("project_id {}".format(project_id)) + logger.debug("project_id {}".format(project_id)) for containers in self.CACHE.containers.values(): - LOG.info("containers {}".format(containers)) + logger.info("containers {}".format(containers)) for container in containers: if container.get("keystone_project_id") == project_id: - if "interface" in container['name']: + if "pipeline" in container['name']: return "http://{}:{}".format( container['name'], container['port']) @@ -80,7 +80,7 @@ class OsloWrapper(Resource): for containers in self.CACHE.containers.values(): for container in containers: if container.get("keystone_project_id") == project_id: - if "interface" in container['name']: + if "pipeline" in container['name']: return "http://{}:{}".format( container['name'], container['port']) @@ -99,11 +99,11 @@ class OsloWrapper(Resource): _object = self.__get_object(target, credentials) _action = rule _project_id = self.__get_project_id(target, credentials) - LOG.debug("POST with args project={} / " + logger.debug("POST with args project={} / " "subject={} - object={} - action={}".format( _project_id, _subject, _object, rule)) interface_url = self.get_interface_url(_project_id) - LOG.debug("interface_url={}".format(interface_url)) + logger.debug("interface_url={}".format(interface_url)) req = requests.get("{}/authz/{}/{}/{}/{}".format( interface_url, _project_id, @@ -111,7 +111,7 @@ class OsloWrapper(Resource): _object, _action )) - LOG.debug("Get interface {}".format(req.text)) + logger.debug("Get interface {}".format(req.text)) if req.status_code == 200: if req.json().get("result", False): return True diff --git a/python_moondb/tests/unit_python/policies/mock_data.py b/python_moondb/tests/unit_python/policies/mock_data.py index b2642979..23eeef64 100644 --- a/python_moondb/tests/unit_python/policies/mock_data.py +++ b/python_moondb/tests/unit_python/policies/mock_data.py @@ -30,6 +30,16 @@ def create_policy(model_id): return value +def create_pdp(pdp_ids): + value = { + "name": "test_pdp", + "security_pipeline": pdp_ids, + "keystone_project_id": "keystone_project_id1", + "description": "...", + } + return value + + def get_policy_id(): import policies.test_policies as test_policies import models.test_models as test_models @@ -39,7 +49,7 @@ def get_policy_id(): model = test_models.add_model(value=create_model(meta_rule_id)) model_id = list(model.keys())[0] value = create_policy(model_id) - policy = test_policies.add_policies(value) + policy = test_policies.add_policies(value=value) assert policy policy_id = list(policy.keys())[0] return policy_id diff --git a/python_moondb/tests/unit_python/policies/test_data.py b/python_moondb/tests/unit_python/policies/test_data.py index 68b1d2a0..875121eb 100755 --- a/python_moondb/tests/unit_python/policies/test_data.py +++ b/python_moondb/tests/unit_python/policies/test_data.py @@ -495,7 +495,7 @@ def test_delete_subject_with_invalid_perimeter_id(db): def test_get_available_metadata(db): policy_id = mock_data.get_policy_id() - metadata = get_available_metadata(policy_id) + metadata = get_available_metadata(policy_id=policy_id) assert metadata assert metadata['object'][0] == "object_category_id1" assert metadata['subject'][0] == "subject_category_id1" @@ -504,10 +504,15 @@ def test_get_available_metadata(db): def test_get_available_metadata_empty_model(db): import policies.test_policies as test_policies - policy_id = mock_data.get_policy_id() value = mock_data.create_policy("invalid") - policy = test_policies.add_policies(value) + policy = test_policies.add_policies(value=value) assert policy policy_id = list(policy.keys())[0] - metadata = get_available_metadata(policy_id) - assert metadata
\ No newline at end of file + metadata = get_available_metadata(policy_id=policy_id) + assert metadata + + +def test_get_available_metadata_with_invalid_policy_id(db): + with pytest.raises(Exception) as exception_info: + get_available_metadata(policy_id='invalid') + assert '400: Policy Unknown' == str(exception_info.value) diff --git a/python_moondb/tests/unit_python/policies/test_policies.py b/python_moondb/tests/unit_python/policies/test_policies.py index acd5d7a8..487cb6a1 100755 --- a/python_moondb/tests/unit_python/policies/test_policies.py +++ b/python_moondb/tests/unit_python/policies/test_policies.py @@ -3,13 +3,16 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. +import pytest +import policies.mock_data as mock_data + def get_policies(): from python_moondb.core import PolicyManager return PolicyManager.get_policies("admin") -def add_policies(value=None): +def add_policies(policy_id=None, value=None): from python_moondb.core import PolicyManager if not value: value = { @@ -18,7 +21,7 @@ def add_policies(value=None): "genre": "authz", "description": "test", } - return PolicyManager.add_policy("admin", value=value) + return PolicyManager.add_policy("admin", policy_id=policy_id, value=value) def delete_policies(uuid=None, name=None): @@ -31,6 +34,16 @@ def delete_policies(uuid=None, name=None): PolicyManager.delete_policy("admin", uuid) +def update_policy(policy_id, value): + from python_moondb.core import PolicyManager + return PolicyManager.update_policy("admin", policy_id, value) + + +def get_policy_from_meta_rules(meta_rule_id): + from python_moondb.core import PolicyManager + return PolicyManager.get_policy_from_meta_rules("admin", meta_rule_id) + + def get_rules(policy_id=None, meta_rule_id=None, rule_id=None): from python_moondb.core import PolicyManager return PolicyManager.get_rules("", policy_id, meta_rule_id, rule_id) @@ -65,7 +78,7 @@ def test_add_policies(db): "genre": "authz", "description": "test", } - policies = add_policies(value) + policies = add_policies(value=value) assert isinstance(policies, dict) assert policies assert len(policies.keys()) == 1 @@ -75,6 +88,20 @@ def test_add_policies(db): assert policies[policy_id][key] == value[key] +def test_add_policies_twice_with_same_id(db): + policy_id = 'policy_id_1' + value = { + "name": "test_policy", + "model_id": "", + "genre": "authz", + "description": "test", + } + add_policies(policy_id, value) + with pytest.raises(Exception) as exception_info: + add_policies(policy_id, value) + assert str(exception_info.value) == '409: Policy Error' + + def test_delete_policies(db): value = { "name": "test_policy1", @@ -82,7 +109,7 @@ def test_delete_policies(db): "genre": "authz", "description": "test", } - policies = add_policies(value) + policies = add_policies(value=value) policy_id1 = list(policies.keys())[0] value = { "name": "test_policy2", @@ -90,7 +117,7 @@ def test_delete_policies(db): "genre": "authz", "description": "test", } - policies = add_policies(value) + policies = add_policies(value=value) policy_id2 = list(policies.keys())[0] assert policy_id1 != policy_id2 delete_policies(policy_id1) @@ -98,6 +125,106 @@ def test_delete_policies(db): assert policy_id1 not in policies +def test_delete_policies_with_invalid_id(db): + policy_id = 'policy_id_1' + with pytest.raises(Exception) as exception_info: + delete_policies(policy_id) + assert str(exception_info.value) == '400: Policy Unknown' + + +def test_update_policy(db): + policies = add_policies() + policy_id = list(policies.keys())[0] + value = { + "name": "test_policy4", + "model_id": "", + "genre": "authz", + "description": "test-3", + } + updated_policy = update_policy(policy_id, value) + assert updated_policy + for key in ("genre", "name", "model_id", "description"): + assert key in updated_policy[policy_id] + assert updated_policy[policy_id][key] == value[key] + + +def test_update_policy_with_invalid_id(db): + policy_id = 'invalid-id' + value = { + "name": "test_policy4", + "model_id": "", + "genre": "authz", + "description": "test-3", + } + with pytest.raises(Exception) as exception_info: + update_policy(policy_id, value) + assert str(exception_info.value) == '400: Policy Unknown' + + +def test_get_policy_from_meta_rules(db): + import models.test_models as test_models + import models.test_meta_rules as test_meta_rules + import test_pdp as test_pdp + meta_rule = test_meta_rules.add_meta_rule(value=mock_data.create_meta_rule()) + meta_rule_id = list(meta_rule.keys())[0] + model = test_models.add_model(value=mock_data.create_model(meta_rule_id)) + model_id = list(model.keys())[0] + value = mock_data.create_policy(model_id) + policy = add_policies(value=value) + assert policy + policy_id = list(policy.keys())[0] + pdp_ids = [policy_id,] + pdp_obj = mock_data.create_pdp(pdp_ids) + test_pdp.add_pdp(value=pdp_obj) + matched_policy_id = get_policy_from_meta_rules(meta_rule_id) + assert matched_policy_id + assert policy_id == matched_policy_id + + +def test_get_policy_from_meta_rules_with_no_policy_ids(db): + import test_pdp as test_pdp + meta_rule_id = 'meta_rule_id' + value = { + "name": "test_pdp", + "security_pipeline": [], + "keystone_project_id": "keystone_project_id1", + "description": "...", + } + test_pdp.add_pdp(value=value) + matched_policy_id = get_policy_from_meta_rules(meta_rule_id) + assert not matched_policy_id + + +def test_get_policy_from_meta_rules_with_no_policies(db): + import test_pdp as test_pdp + meta_rule_id = 'meta_rule_id' + policy_id = 'invalid' + pdp_ids = [policy_id,] + pdp_obj = mock_data.create_pdp(pdp_ids) + test_pdp.add_pdp(value=pdp_obj) + with pytest.raises(Exception) as exception_info: + get_policy_from_meta_rules(meta_rule_id) + assert str(exception_info.value) == '400: Policy Unknown' + + +def test_get_policy_from_meta_rules_with_no_models(db): + import models.test_meta_rules as test_meta_rules + import test_pdp as test_pdp + meta_rule = test_meta_rules.add_meta_rule(value=mock_data.create_meta_rule()) + meta_rule_id = list(meta_rule.keys())[0] + model_id = 'invalid' + value = mock_data.create_policy(model_id) + policy = add_policies(value=value) + assert policy + policy_id = list(policy.keys())[0] + pdp_ids = [policy_id,] + pdp_obj = mock_data.create_pdp(pdp_ids) + test_pdp.add_pdp(value=pdp_obj) + with pytest.raises(Exception) as exception_info: + get_policy_from_meta_rules(meta_rule_id) + assert str(exception_info.value) == '400: Model Unknown' + + def test_get_rules(db): value = { "rule": ("low", "medium", "vm-action"), diff --git a/python_moonutilities/python_moonutilities/configuration.py b/python_moonutilities/python_moonutilities/configuration.py index c31432c3..4a072de4 100644 --- a/python_moonutilities/python_moonutilities/configuration.py +++ b/python_moonutilities/python_moonutilities/configuration.py @@ -25,10 +25,11 @@ def init_logging(): config = get_configuration("logging") logging.config.dictConfig(config['logging']) + def increment_port(): components_object = get_configuration("components/port_start") - if 'port_start' in components_object: - components_port_start = int(get_configuration("components/port_start")['port_start']) + if 'components/port_start' in components_object: + components_port_start = int(components_object['components/port_start']) components_port_start += 1 else: raise exceptions.ConsulComponentContentError("error={}".format(components_object)) @@ -39,6 +40,7 @@ def increment_port(): raise exceptions.ConsulError return components_port_start + def get_configuration(key): url = "http://{}:{}/v1/kv/{}".format(CONSUL_HOST, CONSUL_PORT, key) req = requests.get(url) @@ -62,6 +64,7 @@ def get_configuration(key): } for item in data ] + def add_component(name, uuid, port=None, bind="127.0.0.1", keystone_id="", extra=None, container=None): data = { "hostname": name, @@ -81,29 +84,16 @@ def add_component(name, uuid, port=None, bind="127.0.0.1", keystone_id="", extra logger.debug("data={}".format(data)) raise exceptions.ConsulError logger.info("Add component {}".format(req.text)) - return configuration.get_configuration("components/"+uuid) + return get_configuration("components/"+uuid) + def get_plugins(): - url = "http://{}:{}/v1/kv/plugins?recurse=true".format(CONSUL_HOST, CONSUL_PORT) - req = requests.get(url) - if req.status_code != 200: - logger.info("url={}".format(url)) - raise exceptions.ConsulError - data = req.json() - if len(data) == 1: - data = data[0] - if all(k in data for k in ("Key", "Value")): - return {data["Key"].replace("plugins/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))} - raise exceptions.ConsulComponentContentError("error={}".format(data)) - else: - for item in data: - if not all(k in item for k in ("Key", "Value")): - logger.warning("invalidate content {}".format(item)) - raise exceptions.ConsulComponentContentError("error={}".format(data)) - return { - item["Key"].replace("plugins/", ""): json.loads(base64.b64decode(item["Value"]).decode("utf-8")) - for item in data - } + pipeline = get_configuration("components/pipeline") + logger.debug("pipeline={}".format(pipeline)) + components = pipeline.get("components/pipeline") + components.pop('interface') + return components + def get_components(): url = "http://{}:{}/v1/kv/components?recurse=true".format(CONSUL_HOST, CONSUL_PORT) diff --git a/python_moonutilities/python_moonutilities/exceptions.py b/python_moonutilities/python_moonutilities/exceptions.py index d85cef49..5b9ff340 100644 --- a/python_moonutilities/python_moonutilities/exceptions.py +++ b/python_moonutilities/python_moonutilities/exceptions.py @@ -3,7 +3,7 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -from oslo_log import log as logging +import logging from werkzeug.exceptions import HTTPException logger = logging.getLogger("moon.utilities.exceptions") diff --git a/tools/bin/delete_orchestrator.sh b/tools/bin/delete_orchestrator.sh index 9b531e22..4d9d7c98 100644 --- a/tools/bin/delete_orchestrator.sh +++ b/tools/bin/delete_orchestrator.sh @@ -4,21 +4,19 @@ set +x kubectl delete -n moon -f tools/moon_kubernetes/templates/moon_orchestrator.yaml for i in $(kubectl get deployments -n moon | grep wrapper | cut -d " " -f 1 | xargs); do + echo deleting $i kubectl delete deployments/$i -n moon; done -for i in $(kubectl get deployments -n moon | grep interface | cut -d " " -f 1 | xargs); do - kubectl delete deployments/$i -n moon; -done -for i in $(kubectl get deployments -n moon | grep authz | cut -d " " -f 1 | xargs); do +for i in $(kubectl get deployments -n moon | grep pipeline | cut -d " " -f 1 | xargs); do + echo deleting $i kubectl delete deployments/$i -n moon; done for i in $(kubectl get services -n moon | grep wrapper | cut -d " " -f 1 | xargs); do + echo deleting $i kubectl delete services/$i -n moon; done -for i in $(kubectl get services -n moon | grep interface | cut -d " " -f 1 | xargs); do - kubectl delete services/$i -n moon; -done -for i in $(kubectl get services -n moon | grep authz | cut -d " " -f 1 | xargs); do +for i in $(kubectl get services -n moon | grep pipeline | cut -d " " -f 1 | xargs); do + echo deleting $i kubectl delete services/$i -n moon; done diff --git a/tools/moon_kubernetes/conf/moon.conf b/tools/moon_kubernetes/conf/moon.conf index a5a40ad2..cf3f5c58 100644 --- a/tools/moon_kubernetes/conf/moon.conf +++ b/tools/moon_kubernetes/conf/moon.conf @@ -14,20 +14,23 @@ openstack: external: url: http://keystone:30006/v3 -plugins: - authz: - container: wukongsun/moon_authz:v4.3 - port: 8081 - session: - container: asteroide/session:latest - port: 8082 - components: - interface: - port: 8080 - bind: 0.0.0.0 - hostname: interface - container: wukongsun/moon_interface:v4.3 + port_start: + 31001 + pipeline: + interface: + port: 8080 + bind: 0.0.0.0 + hostname: interface + container: wukongsun/moon_interface:v4.3 + authz: + port: 8081 + bind: 0.0.0.0 + hostname: interface + container: wukongsun/moon_authz:v4.3 + session: + container: asteroide/session:latest + port: 8082 orchestrator: port: 8083 bind: 0.0.0.0 |