From 295ba85cba341338a2e74fef729dc74d091ac778 Mon Sep 17 00:00:00 2001 From: Thomas Duval Date: Thu, 4 Jan 2018 14:41:56 +0100 Subject: update interface pod name to pipeline Change-Id: I6a5a9ce3a27ea3ea40735dffa5f6a713f571cdc8 --- moon_authz/moon_authz/api/authorization.py | 2 +- moon_authz/moon_authz/server.py | 17 +++++----- moon_interface/moon_interface/server.py | 11 ++++--- moon_orchestrator/moon_orchestrator/http_server.py | 11 +++---- moon_orchestrator/tests/unit_python/utilities.py | 2 +- moon_wrapper/moon_wrapper/api/generic.py | 6 ++-- moon_wrapper/moon_wrapper/api/oslowrapper.py | 20 ++++++------ .../python_moonutilities/configuration.py | 36 ++++++++-------------- .../python_moonutilities/exceptions.py | 2 +- tools/bin/delete_orchestrator.sh | 14 ++++----- 10 files changed, 56 insertions(+), 65 deletions(-) diff --git a/moon_authz/moon_authz/api/authorization.py b/moon_authz/moon_authz/api/authorization.py index c83dd72c..ea177d81 100644 --- a/moon_authz/moon_authz/api/authorization.py +++ b/moon_authz/moon_authz/api/authorization.py @@ -73,7 +73,7 @@ class Authz(Resource): return response def run(self): - logger.info("self.context.pdp_set={}".format(self.context.pdp_set)) + logger.debug("self.context.pdp_set={}".format(self.context.pdp_set)) result, message = self.__check_rules() if result: return self.__exec_instructions(result) diff --git a/moon_authz/moon_authz/server.py b/moon_authz/moon_authz/server.py index 8715bd87..0cc5f6fc 100644 --- a/moon_authz/moon_authz/server.py +++ b/moon_authz/moon_authz/server.py @@ -6,7 +6,7 @@ import os import logging from moon_authz.http_server import HTTPServer as Server -from python_moonutilities import configuration +from python_moonutilities import configuration, exceptions logger = logging.getLogger("moon.authz.server") @@ -21,12 +21,15 @@ def create_server(): meta_rule_id = os.getenv("META_RULE_ID") keystone_project_id = os.getenv("KEYSTONE_PROJECT_ID") logger.info("component_type={}".format(component_type)) - conf = configuration.get_configuration("plugins/{}".format(component_type)) - conf["plugins/{}".format(component_type)]['id'] = component_id - hostname = conf["plugins/{}".format(component_type)].get('hostname', - component_id) - port = conf["plugins/{}".format(component_type)].get('port', tcp_port) - bind = conf["plugins/{}".format(component_type)].get('bind', "0.0.0.0") + conf = configuration.get_plugins() + # conf = configuration.get_configuration("plugins/{}".format(component_type)) + # conf["plugins/{}".format(component_type)]['id'] = component_id + if component_type not in conf: + raise exceptions.ConsulComponentNotFound("{} not found".format( + component_type)) + hostname = conf[component_type].get('hostname', component_id) + port = conf[component_type].get('port', tcp_port) + bind = conf[component_type].get('bind', "0.0.0.0") logger.info("Starting server with IP {} on port {} bind to {}".format( hostname, port, bind)) diff --git a/moon_interface/moon_interface/server.py b/moon_interface/moon_interface/server.py index 13955c3e..0af1fd06 100644 --- a/moon_interface/moon_interface/server.py +++ b/moon_interface/moon_interface/server.py @@ -13,15 +13,16 @@ logger = logging.getLogger("moon.interface.server") def create_server(): configuration.init_logging() try: - conf = configuration.get_configuration("components/interface") - hostname = conf["components/interface"].get("hostname", "interface") - port = conf["components/interface"].get("port", 80) - bind = conf["components/interface"].get("bind", "127.0.0.1") + conf = configuration.get_configuration("components/pipeline").get( + "components/pipeline", {}).get("interface", {}) + hostname = conf.get("hostname", "pipeline") + port = conf.get("port", 80) + bind = conf.get("bind", "127.0.0.1") except exceptions.ConsulComponentNotFound: hostname = "interface" bind = "127.0.0.1" port = 80 - configuration.add_component(uuid="interface", + configuration.add_component(uuid="pipeline", name=hostname, port=port, bind=bind) diff --git a/moon_orchestrator/moon_orchestrator/http_server.py b/moon_orchestrator/moon_orchestrator/http_server.py index 7105ea7a..00be0335 100644 --- a/moon_orchestrator/moon_orchestrator/http_server.py +++ b/moon_orchestrator/moon_orchestrator/http_server.py @@ -179,7 +179,6 @@ class HTTPServer(Server): logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) - # TODO: get data from consul data = [{ "name": hostname + "-" + get_random_name(), "container": container, @@ -217,15 +216,15 @@ class HTTPServer(Server): return plugins = configuration.get_plugins() - conf = configuration.get_configuration("components/interface") - i_hostname = conf["components/interface"].get("hostname", "interface") - i_port = conf["components/interface"].get("port", 80) - i_container = conf["components/interface"].get( + conf = configuration.get_configuration("components/pipeline") + # i_hostname = conf["components/pipeline"].get("interface").get("hostname", "interface") + i_port = conf["components/pipeline"].get("interface").get("port", 80) + i_container = conf["components/pipeline"].get("interface").get( "container", "wukongsun/moon_interface:v4.3") data = [ { - "name": i_hostname + "-" + get_random_name(), + "name": "pipeline-" + get_random_name(), "container": i_container, "port": i_port, 'pdp_id': pdp_id, diff --git a/moon_orchestrator/tests/unit_python/utilities.py b/moon_orchestrator/tests/unit_python/utilities.py index aec03d9d..d64e4c7b 100644 --- a/moon_orchestrator/tests/unit_python/utilities.py +++ b/moon_orchestrator/tests/unit_python/utilities.py @@ -144,7 +144,7 @@ COMPONENTS = ( "slave", "components/manager", "components/orchestrator", - "components/interface", + "components/pipeline", "components/wrapper", ) diff --git a/moon_wrapper/moon_wrapper/api/generic.py b/moon_wrapper/moon_wrapper/api/generic.py index 7dd44fb4..498513c7 100644 --- a/moon_wrapper/moon_wrapper/api/generic.py +++ b/moon_wrapper/moon_wrapper/api/generic.py @@ -7,13 +7,13 @@ Those API are helping API used to manage the Moon platform. """ from flask_restful import Resource, request -from oslo_log import log as logging +import logging import moon_wrapper.api from python_moonutilities.security_functions import check_auth __version__ = "0.1.0" -LOG = logging.getLogger("moon.manager.api." + __name__) +logger = logging.getLogger("moon.manager.api." + __name__) class Status(Resource): @@ -125,7 +125,7 @@ class API(Resource): if endpoint_id in api_desc[group_id]: return {group_id: {endpoint_id: api_desc[group_id][endpoint_id]}} elif len(endpoint_id) > 0: - LOG.error("Unknown endpoint_id {}".format(endpoint_id)) + logger.error("Unknown endpoint_id {}".format(endpoint_id)) return {"error": "Unknown endpoint_id {}".format(endpoint_id)} return {group_id: api_desc[group_id]} return api_desc diff --git a/moon_wrapper/moon_wrapper/api/oslowrapper.py b/moon_wrapper/moon_wrapper/api/oslowrapper.py index a422ee42..03bdfc69 100644 --- a/moon_wrapper/moon_wrapper/api/oslowrapper.py +++ b/moon_wrapper/moon_wrapper/api/oslowrapper.py @@ -16,7 +16,7 @@ from python_moonutilities import exceptions __version__ = "0.1.0" -LOG = logging.getLogger("moon.wrapper.api." + __name__) +logger = logging.getLogger("moon.wrapper.api." + __name__) class OsloWrapper(Resource): @@ -35,7 +35,7 @@ class OsloWrapper(Resource): self.TIMEOUT = 5 def post(self): - LOG.debug("POST {}".format(request.form)) + logger.debug("POST {}".format(request.form)) response = flask.make_response("False") if self.manage_data(): response = flask.make_response("True") @@ -62,16 +62,16 @@ class OsloWrapper(Resource): @staticmethod def __get_project_id(target, credentials): - LOG.info("__get_project_id {}".format(target)) + logger.info("__get_project_id {}".format(target)) return target.get("project_id", "none") def get_interface_url(self, project_id): - LOG.info("project_id {}".format(project_id)) + logger.debug("project_id {}".format(project_id)) for containers in self.CACHE.containers.values(): - LOG.info("containers {}".format(containers)) + logger.info("containers {}".format(containers)) for container in containers: if container.get("keystone_project_id") == project_id: - if "interface" in container['name']: + if "pipeline" in container['name']: return "http://{}:{}".format( container['name'], container['port']) @@ -80,7 +80,7 @@ class OsloWrapper(Resource): for containers in self.CACHE.containers.values(): for container in containers: if container.get("keystone_project_id") == project_id: - if "interface" in container['name']: + if "pipeline" in container['name']: return "http://{}:{}".format( container['name'], container['port']) @@ -99,11 +99,11 @@ class OsloWrapper(Resource): _object = self.__get_object(target, credentials) _action = rule _project_id = self.__get_project_id(target, credentials) - LOG.debug("POST with args project={} / " + logger.debug("POST with args project={} / " "subject={} - object={} - action={}".format( _project_id, _subject, _object, rule)) interface_url = self.get_interface_url(_project_id) - LOG.debug("interface_url={}".format(interface_url)) + logger.debug("interface_url={}".format(interface_url)) req = requests.get("{}/authz/{}/{}/{}/{}".format( interface_url, _project_id, @@ -111,7 +111,7 @@ class OsloWrapper(Resource): _object, _action )) - LOG.debug("Get interface {}".format(req.text)) + logger.debug("Get interface {}".format(req.text)) if req.status_code == 200: if req.json().get("result", False): return True diff --git a/python_moonutilities/python_moonutilities/configuration.py b/python_moonutilities/python_moonutilities/configuration.py index c31432c3..4a072de4 100644 --- a/python_moonutilities/python_moonutilities/configuration.py +++ b/python_moonutilities/python_moonutilities/configuration.py @@ -25,10 +25,11 @@ def init_logging(): config = get_configuration("logging") logging.config.dictConfig(config['logging']) + def increment_port(): components_object = get_configuration("components/port_start") - if 'port_start' in components_object: - components_port_start = int(get_configuration("components/port_start")['port_start']) + if 'components/port_start' in components_object: + components_port_start = int(components_object['components/port_start']) components_port_start += 1 else: raise exceptions.ConsulComponentContentError("error={}".format(components_object)) @@ -39,6 +40,7 @@ def increment_port(): raise exceptions.ConsulError return components_port_start + def get_configuration(key): url = "http://{}:{}/v1/kv/{}".format(CONSUL_HOST, CONSUL_PORT, key) req = requests.get(url) @@ -62,6 +64,7 @@ def get_configuration(key): } for item in data ] + def add_component(name, uuid, port=None, bind="127.0.0.1", keystone_id="", extra=None, container=None): data = { "hostname": name, @@ -81,29 +84,16 @@ def add_component(name, uuid, port=None, bind="127.0.0.1", keystone_id="", extra logger.debug("data={}".format(data)) raise exceptions.ConsulError logger.info("Add component {}".format(req.text)) - return configuration.get_configuration("components/"+uuid) + return get_configuration("components/"+uuid) + def get_plugins(): - url = "http://{}:{}/v1/kv/plugins?recurse=true".format(CONSUL_HOST, CONSUL_PORT) - req = requests.get(url) - if req.status_code != 200: - logger.info("url={}".format(url)) - raise exceptions.ConsulError - data = req.json() - if len(data) == 1: - data = data[0] - if all(k in data for k in ("Key", "Value")): - return {data["Key"].replace("plugins/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))} - raise exceptions.ConsulComponentContentError("error={}".format(data)) - else: - for item in data: - if not all(k in item for k in ("Key", "Value")): - logger.warning("invalidate content {}".format(item)) - raise exceptions.ConsulComponentContentError("error={}".format(data)) - return { - item["Key"].replace("plugins/", ""): json.loads(base64.b64decode(item["Value"]).decode("utf-8")) - for item in data - } + pipeline = get_configuration("components/pipeline") + logger.debug("pipeline={}".format(pipeline)) + components = pipeline.get("components/pipeline") + components.pop('interface') + return components + def get_components(): url = "http://{}:{}/v1/kv/components?recurse=true".format(CONSUL_HOST, CONSUL_PORT) diff --git a/python_moonutilities/python_moonutilities/exceptions.py b/python_moonutilities/python_moonutilities/exceptions.py index d85cef49..5b9ff340 100644 --- a/python_moonutilities/python_moonutilities/exceptions.py +++ b/python_moonutilities/python_moonutilities/exceptions.py @@ -3,7 +3,7 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -from oslo_log import log as logging +import logging from werkzeug.exceptions import HTTPException logger = logging.getLogger("moon.utilities.exceptions") diff --git a/tools/bin/delete_orchestrator.sh b/tools/bin/delete_orchestrator.sh index 9b531e22..4d9d7c98 100644 --- a/tools/bin/delete_orchestrator.sh +++ b/tools/bin/delete_orchestrator.sh @@ -4,21 +4,19 @@ set +x kubectl delete -n moon -f tools/moon_kubernetes/templates/moon_orchestrator.yaml for i in $(kubectl get deployments -n moon | grep wrapper | cut -d " " -f 1 | xargs); do + echo deleting $i kubectl delete deployments/$i -n moon; done -for i in $(kubectl get deployments -n moon | grep interface | cut -d " " -f 1 | xargs); do - kubectl delete deployments/$i -n moon; -done -for i in $(kubectl get deployments -n moon | grep authz | cut -d " " -f 1 | xargs); do +for i in $(kubectl get deployments -n moon | grep pipeline | cut -d " " -f 1 | xargs); do + echo deleting $i kubectl delete deployments/$i -n moon; done for i in $(kubectl get services -n moon | grep wrapper | cut -d " " -f 1 | xargs); do + echo deleting $i kubectl delete services/$i -n moon; done -for i in $(kubectl get services -n moon | grep interface | cut -d " " -f 1 | xargs); do - kubectl delete services/$i -n moon; -done -for i in $(kubectl get services -n moon | grep authz | cut -d " " -f 1 | xargs); do +for i in $(kubectl get services -n moon | grep pipeline | cut -d " " -f 1 | xargs); do + echo deleting $i kubectl delete services/$i -n moon; done -- cgit 1.2.3-korg