From 9519d179cda8734fa0890d136c4bd2019bb7ddc4 Mon Sep 17 00:00:00 2001 From: Thomas Duval Date: Mon, 6 Nov 2017 14:02:18 +0100 Subject: Fix the bug on the connection between interface and authz. Change-Id: Iea2f763fb2cfb90250be76d91cb2fe0e9b481716 --- moonv4/kubernetes/conf/ports.conf | 4 +++ moonv4/moon_authz/Dockerfile | 2 +- moonv4/moon_interface/Dockerfile | 2 +- .../moon_interface/authz_requests.py | 29 ++++++++--------- moonv4/moon_manager/Dockerfile | 2 +- moonv4/moon_manager/moon_manager/api/pdp.py | 14 ++++++-- .../moon_orchestrator/api/pods.py | 11 ++++++- .../moon_orchestrator/moon_orchestrator/drivers.py | 24 +++++++------- .../moon_orchestrator/http_server.py | 37 ++++++++++++++++------ moonv4/moon_utilities/Changelog | 4 +++ moonv4/moon_utilities/build.sh | 2 +- moonv4/moon_utilities/moon_utilities/__init__.py | 2 +- moonv4/moon_utilities/moon_utilities/cache.py | 1 + moonv4/moon_wrapper/Dockerfile | 2 +- moonv4/templates/moonforming/moon.conf | 4 ++- 15 files changed, 92 insertions(+), 48 deletions(-) (limited to 'moonv4') diff --git a/moonv4/kubernetes/conf/ports.conf b/moonv4/kubernetes/conf/ports.conf index d1b12aac..487945c0 100644 --- a/moonv4/kubernetes/conf/ports.conf +++ b/moonv4/kubernetes/conf/ports.conf @@ -18,3 +18,7 @@ keystone: wrapper: port: 8080 kport: 30010 +interface: + port: 8080 +authz: + port: 8081 diff --git a/moonv4/moon_authz/Dockerfile b/moonv4/moon_authz/Dockerfile index 7ab172b0..4189c333 100644 --- a/moonv4/moon_authz/Dockerfile +++ b/moonv4/moon_authz/Dockerfile @@ -6,7 +6,7 @@ RUN pip3 install pip --upgrade ADD . /root WORKDIR /root/ RUN pip3 install -r requirements.txt --upgrade -RUN pip3 install /root/dist/* --upgrade +#RUN pip3 install /root/dist/* --upgrade RUN pip3 install . CMD ["python3", "-m", "moon_authz"] \ No newline at end of file diff --git a/moonv4/moon_interface/Dockerfile b/moonv4/moon_interface/Dockerfile index 1bc7e87c..268aba48 100644 --- a/moonv4/moon_interface/Dockerfile +++ b/moonv4/moon_interface/Dockerfile @@ -6,7 +6,7 @@ RUN pip3 install moon_utilities moon_db pip --upgrade ADD . /root WORKDIR /root/ RUN pip3 install -r requirements.txt --upgrade -RUN pip3 install /root/dist/* --upgrade +#RUN pip3 install /root/dist/* --upgrade RUN pip3 install . CMD ["python3", "-m", "moon_interface"] \ No newline at end of file diff --git a/moonv4/moon_interface/moon_interface/authz_requests.py b/moonv4/moon_interface/moon_interface/authz_requests.py index 10356787..b6b0e58e 100644 --- a/moonv4/moon_interface/moon_interface/authz_requests.py +++ b/moonv4/moon_interface/moon_interface/authz_requests.py @@ -28,8 +28,6 @@ class AuthzRequest: self.context = Context(ctx, CACHE) self.args = args self.request_id = ctx["request_id"] - # LOG.info("container={}".format(CACHE.containers)) - # LOG.info("container_chaining={}".format(CACHE.container_chaining)) if ctx['project_id'] not in CACHE.container_chaining: raise exceptions.KeystoneProjectError("Unknown Project ID {}".format(ctx['project_id'])) self.container_chaining = CACHE.container_chaining[ctx['project_id']] @@ -40,36 +38,35 @@ class AuthzRequest: def run(self): self.context.delete_cache() + req = None try: - LOG.debug("url=http://{}:{}/authz".format( - self.container_chaining[0]["hostname"], - self.container_chaining[0]["port"])) req = requests.post("http://{}:{}/authz".format( - self.container_chaining[0]["hostname"], + self.container_chaining[0]["hostip"], self.container_chaining[0]["port"], ), data=pickle.dumps(self.context)) if req.status_code != 200: - # LOG.error("Cannot connect to {}".format( - # "http://{}:{}/authz".format( - # self.container_chaining[0]["hostname"], - # self.container_chaining[0]["port"] - # ))) + # LOG.error("req={}".format(req)) raise exceptions.AuthzException( "Receive bad response from Authz function " - "(with hostname - {})".format( + "(with IP address - {})".format( req.status_code )) except requests.exceptions.ConnectionError: + LOG.error("Cannot connect to {}".format( + "http://{}:{}/authz".format( + self.container_chaining[0]["hostip"], + self.container_chaining[0]["port"] + ))) + except ValueError: try: req = requests.post("http://{}:{}/authz".format( - self.container_chaining[0]["hostip"], + self.container_chaining[0]["hostname"], self.container_chaining[0]["port"], ), data=pickle.dumps(self.context)) if req.status_code != 200: - # LOG.error("req={}".format(req)) raise exceptions.AuthzException( "Receive bad response from Authz function " - "(with IP address - {})".format( + "(with hostname - {})".format( req.status_code )) except requests.exceptions.ConnectionError: @@ -81,7 +78,7 @@ class AuthzRequest: raise exceptions.AuthzException( "Cannot connect to Authz function with IP address") self.context.set_cache(CACHE) - if len(self.container_chaining) == 1: + if req and len(self.container_chaining) == 1: # req.raw.decode_content = True self.result = pickle.loads(req.content) diff --git a/moonv4/moon_manager/Dockerfile b/moonv4/moon_manager/Dockerfile index 17bee16a..873e3aa2 100644 --- a/moonv4/moon_manager/Dockerfile +++ b/moonv4/moon_manager/Dockerfile @@ -6,7 +6,7 @@ RUN pip3 install pip --upgrade ADD . /root WORKDIR /root/ RUN pip3 install -r requirements.txt -RUN pip3 install /root/dist/* --upgrade +#RUN pip3 install /root/dist/* --upgrade RUN pip3 install . CMD ["python3", "-m", "moon_manager"] \ No newline at end of file diff --git a/moonv4/moon_manager/moon_manager/api/pdp.py b/moonv4/moon_manager/moon_manager/api/pdp.py index ffc91fba..465f4dba 100644 --- a/moonv4/moon_manager/moon_manager/api/pdp.py +++ b/moonv4/moon_manager/moon_manager/api/pdp.py @@ -14,6 +14,8 @@ import requests import time from moon_utilities.security_functions import check_auth from moon_db.core import PDPManager +from moon_db.core import PolicyManager +from moon_db.core import ModelManager from moon_utilities import configuration __version__ = "0.1.0" @@ -26,15 +28,23 @@ def delete_pod(uuid): def add_pod(uuid, data): + if not data.get("keystone_project_id"): + return LOG.info("Add a new pod {}".format(data)) + if "pdp_id" not in data: + data["pdp_id"] = uuid + data['policies'] = PolicyManager.get_policies(user_id="admin") + data['models'] = ModelManager.get_models(user_id="admin") conf = configuration.get_configuration("components/orchestrator") hostname = conf["components/orchestrator"].get("hostname", "orchestrator") port = conf["components/orchestrator"].get("port", 80) proto = conf["components/orchestrator"].get("protocol", "http") while True: try: - req = requests.post("{}://{}:{}/pods".format(proto, hostname, port), - data=data) + req = requests.post( + "{}://{}:{}/pods".format(proto, hostname, port), + json=data, + headers={"content-type": "application/json"}) except requests.exceptions.ConnectionError: LOG.warning("Orchestrator is not ready, standby...") time.sleep(1) diff --git a/moonv4/moon_orchestrator/moon_orchestrator/api/pods.py b/moonv4/moon_orchestrator/moon_orchestrator/api/pods.py index a7ca1cbc..c5c5b2c4 100644 --- a/moonv4/moon_orchestrator/moon_orchestrator/api/pods.py +++ b/moonv4/moon_orchestrator/moon_orchestrator/api/pods.py @@ -25,6 +25,7 @@ class Pods(Resource): def __init__(self, **kwargs): self.driver = kwargs.get("driver") + self.create_security_function = kwargs.get("create_security_function_hook") @check_auth def get(self, uuid=None, user_id=None): @@ -72,7 +73,15 @@ class Pods(Resource): } } """ - return {"pods": None} + LOG.info("POST param={}".format(request.json)) + self.create_security_function( + request.json.get("keystone_project_id"), + request.json.get("pdp_id"), + request.json.get("security_pipeline"), + manager_data=request.json, + active_context=None, + active_context_name=None) + return {"pods": self.driver.get_pods(request.json.get("pdp_id"))} @check_auth def delete(self, uuid=None, user_id=None): diff --git a/moonv4/moon_orchestrator/moon_orchestrator/drivers.py b/moonv4/moon_orchestrator/moon_orchestrator/drivers.py index 63631c00..63ca8f3c 100644 --- a/moonv4/moon_orchestrator/moon_orchestrator/drivers.py +++ b/moonv4/moon_orchestrator/moon_orchestrator/drivers.py @@ -8,14 +8,14 @@ import logging import urllib3.exceptions from moon_utilities import configuration -logger = logging.getLogger("moon.orchestrator.drivers") +LOG = logging.getLogger("moon.orchestrator.drivers") def get_driver(): try: return K8S() except urllib3.exceptions.MaxRetryError as e: - logger.exception(e) + LOG.exception(e) return Docker() @@ -60,12 +60,12 @@ class K8S(Driver): if name: pods = self.client.list_pod_for_all_namespaces(watch=False) for pod in pods.items: - logger.info("get_pods {}".format(pod.metadata.name)) + LOG.info("get_pods {}".format(pod.metadata.name)) if name in pod.metadata.name: return pod else: return None - logger.info("get_pods cache={}".format(self.cache)) + LOG.info("get_pods cache={}".format(self.cache)) return self.cache @staticmethod @@ -101,7 +101,7 @@ class K8S(Driver): {'name': "TYPE", "value": _data.get('genre', "None")}, {'name': "PORT", "value": str(_data.get('port', 80))}, {'name': "PDP_ID", "value": _data.get('pdp_id', "None")}, - {'name': "META_RULE_ID", "value": "None"}, + {'name': "META_RULE_ID", "value": _data.get('meta_rule_id', "None")}, {'name': "KEYSTONE_PROJECT_ID", "value": _data.get('keystone_project_id', "None")}, ] @@ -109,7 +109,7 @@ class K8S(Driver): ) resp = client.create_namespaced_deployment(body=pod_manifest, namespace='moon') - logger.info("Pod {} created!".format(data[0].get('name'))) + LOG.info("Pod {} created!".format(data[0].get('name'))) # logger.info(yaml.dump(pod_manifest, sys.stdout)) # logger.info(resp) return resp @@ -131,7 +131,7 @@ class K8S(Driver): 'selector': { 'app': data.get('name') }, - 'type': 'NodePort', + # 'type': 'NodePort', 'endpoints': [{ 'port': data.get('port', 80), 'protocol': 'TCP', @@ -144,7 +144,7 @@ class K8S(Driver): service_manifest['spec']['type'] = "NodePort" resp = client.create_namespaced_service(namespace="moon", body=service_manifest) - logger.info("Service {} created!".format(data.get('name'))) + LOG.info("Service {} created!".format(data.get('name'))) return resp def load_pod(self, data, api_client=None, ext_client=None, expose=False): @@ -152,12 +152,12 @@ class K8S(Driver): pod = self.__create_pod(client=ext_client, data=data) service = self.__create_service(client=_client, data=data[0], expose=expose) - # logger.info("load_poad data={}".format(data)) + # logger.info("load_pod data={}".format(data)) # logger.info("pod.metadata.uid={}".format(pod.metadata.uid)) self.cache[pod.metadata.uid] = data def delete_pod(self, uuid=None, name=None): - logger.info("Deleting pod {}".format(uuid)) + LOG.info("Deleting pod {}".format(uuid)) # TODO: delete_namespaced_deployment # https://github.com/kubernetes-incubator/client-python/blob/master/kubernetes/client/apis/extensions_v1beta1_api.py @@ -169,9 +169,9 @@ class K8S(Driver): class Docker(Driver): def load_pod(self, data, api_client=None, ext_client=None): - logger.info("Creating pod {}".format(data[0].get('name'))) + LOG.info("Creating pod {}".format(data[0].get('name'))) raise NotImplementedError def delete_pod(self, uuid=None, name=None): - logger.info("Deleting pod {}".format(uuid)) + LOG.info("Deleting pod {}".format(uuid)) raise NotImplementedError diff --git a/moonv4/moon_orchestrator/moon_orchestrator/http_server.py b/moonv4/moon_orchestrator/moon_orchestrator/http_server.py index 70c5e2d2..c9816f5b 100644 --- a/moonv4/moon_orchestrator/moon_orchestrator/http_server.py +++ b/moonv4/moon_orchestrator/moon_orchestrator/http_server.py @@ -154,6 +154,8 @@ class HTTPServer(Server): self.api.add_resource(Pods, *Pods.__urls__, resource_class_kwargs={ "driver": self.driver, + "create_security_function_hook": + self.create_security_function, }) def run(self): @@ -190,7 +192,8 @@ class HTTPServer(Server): LOG.debug('wrapper pod={}'.format(pod)) def create_security_function(self, keystone_project_id, - pdp_id, policy_ids, active_context=None, + pdp_id, policy_ids, manager_data={}, + active_context=None, active_context_name=None): """ Create security functions @@ -203,13 +206,14 @@ class HTTPServer(Server): security function in all context (ie, in all slaves) :return: None """ - LOG.debug(self.driver.get_pods()) + # LOG.info(self.driver.get_pods()) for key, value in self.driver.get_pods().items(): for _pod in value: if _pod.get('keystone_project_id') == keystone_project_id: LOG.warning("A pod for this Keystone project {} " "already exists.".format(keystone_project_id)) return + plugins = configuration.get_plugins() conf = configuration.get_configuration("components/interface") i_hostname = conf["components/interface"].get("hostname", "interface") @@ -228,12 +232,21 @@ class HTTPServer(Server): "namespace": "moon" }, ] - policies = requests.get("http://{}:{}/policies".format( - self.manager_hostname, self.manager_port)).json().get( - "policies", dict()) - models = requests.get("http://{}:{}/models".format( - self.manager_hostname, self.manager_port)).json().get( - "models", dict()) + LOG.info("data={}".format(data)) + policies = manager_data.get('policies') + if not policies: + LOG.info("No policy data from Manager, trying to get them") + policies = requests.get("http://{}:{}/policies".format( + self.manager_hostname, self.manager_port)).json().get( + "policies", dict()) + LOG.info("policies={}".format(policies)) + models = manager_data.get('models') + if not models: + LOG.info("No models data from Manager, trying to get them") + models = requests.get("http://{}:{}/models".format( + self.manager_hostname, self.manager_port)).json().get( + "models", dict()) + LOG.info("models={}".format(models)) for policy_id in policy_ids: if policy_id in policies: @@ -251,7 +264,10 @@ class HTTPServer(Server): 'keystone_project_id': keystone_project_id, "namespace": "moon" }) + LOG.info("data={}".format(data)) contexts, _active_context = self.driver.get_slaves() + LOG.info("active_context_name={}".format(active_context_name)) + LOG.info("active_context={}".format(active_context)) if active_context_name: for _context in contexts: if _context["name"] == active_context_name: @@ -264,13 +280,14 @@ class HTTPServer(Server): LOG.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) - self.driver.load_pod(data, api_client, ext_client) + self.driver.load_pod(data, api_client, ext_client, expose=False) return + LOG.info("contexts={}".format(contexts)) for _ctx in contexts: _config = config.new_client_from_config(context=_ctx['name']) LOG.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) - self.driver.load_pod(data, api_client, ext_client) + self.driver.load_pod(data, api_client, ext_client, expose=False) diff --git a/moonv4/moon_utilities/Changelog b/moonv4/moon_utilities/Changelog index 185e81dc..ca67cbd4 100644 --- a/moonv4/moon_utilities/Changelog +++ b/moonv4/moon_utilities/Changelog @@ -51,3 +51,7 @@ CHANGES ----- - Update the cache +1.3.4 +----- +- Fix a bug on the connection between interface and authz + diff --git a/moonv4/moon_utilities/build.sh b/moonv4/moon_utilities/build.sh index 79bd9132..b327e49d 100644 --- a/moonv4/moon_utilities/build.sh +++ b/moonv4/moon_utilities/build.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERSION=moon_utilities-1.3.3 +VERSION=moon_utilities-1.3.4 python3 setup.py sdist bdist_wheel diff --git a/moonv4/moon_utilities/moon_utilities/__init__.py b/moonv4/moon_utilities/moon_utilities/__init__.py index 327860b3..3e59b6a0 100644 --- a/moonv4/moon_utilities/moon_utilities/__init__.py +++ b/moonv4/moon_utilities/moon_utilities/__init__.py @@ -3,4 +3,4 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -__version__ = "1.3.3" +__version__ = "1.3.4" diff --git a/moonv4/moon_utilities/moon_utilities/cache.py b/moonv4/moon_utilities/moon_utilities/cache.py index e1d2477a..d62c9d54 100644 --- a/moonv4/moon_utilities/moon_utilities/cache.py +++ b/moonv4/moon_utilities/moon_utilities/cache.py @@ -520,6 +520,7 @@ class Cache(object): "policy_id": policy_id, "meta_rule_id": meta_rule_id, "hostname": container_value["name"], + "hostip": "127.0.0.1", "port": container_value["port"], } ) diff --git a/moonv4/moon_wrapper/Dockerfile b/moonv4/moon_wrapper/Dockerfile index 958f136c..55e7208d 100644 --- a/moonv4/moon_wrapper/Dockerfile +++ b/moonv4/moon_wrapper/Dockerfile @@ -6,7 +6,7 @@ RUN pip3 install pip --upgrade ADD . /root WORKDIR /root/ RUN pip3 install -r requirements.txt --upgrade -#RUN pip3 install /root/dist/* --upgrade +RUN pip3 install /root/dist/* --upgrade RUN pip3 install . CMD ["python3", "-m", "moon_wrapper"] diff --git a/moonv4/templates/moonforming/moon.conf b/moonv4/templates/moonforming/moon.conf index 0d3f740f..2fa1db55 100644 --- a/moonv4/templates/moonforming/moon.conf +++ b/moonv4/templates/moonforming/moon.conf @@ -29,12 +29,14 @@ openstack: plugins: authz: container: wukongsun/moon_authz:v4.3 + port: 8081 session: container: asteroide/session:latest + port: 8082 components: interface: - port: 8081 + port: 8080 bind: 0.0.0.0 hostname: interface container: wukongsun/moon_interface:v4.3 -- cgit 1.2.3-korg