Merge "Fix the bug on the connection between interface and authz."

15 files changed, 92 insertions, 48 deletions

diff --git a/moonv4/kubernetes/conf/ports.conf b/moonv4/kubernetes/conf/ports.conf
index d1b12aac..487945c0 100644
--- a/moonv4/kubernetes/conf/ports.conf
+++ b/moonv4/kubernetes/conf/ports.conf
@@ -18,3 +18,7 @@ keystone:
 wrapper:
     port: 8080
     kport: 30010
+interface:
+    port: 8080
+authz:
+    port: 8081
diff --git a/moonv4/moon_authz/Dockerfile b/moonv4/moon_authz/Dockerfile
index 7ab172b0..4189c333 100644
--- a/moonv4/moon_authz/Dockerfile
+++ b/moonv4/moon_authz/Dockerfile
@@ -6,7 +6,7 @@ RUN pip3 install pip --upgrade
 ADD . /root
 WORKDIR /root/
 RUN pip3 install -r requirements.txt --upgrade
-RUN pip3 install /root/dist/* --upgrade
+#RUN pip3 install /root/dist/* --upgrade
 RUN pip3 install .
 
 CMD ["python3", "-m", "moon_authz"]
\ No newline at end of file
diff --git a/moonv4/moon_interface/Dockerfile b/moonv4/moon_interface/Dockerfile
index 1bc7e87c..268aba48 100644
--- a/moonv4/moon_interface/Dockerfile
+++ b/moonv4/moon_interface/Dockerfile
@@ -6,7 +6,7 @@ RUN pip3 install moon_utilities moon_db pip --upgrade
 ADD . /root
 WORKDIR /root/
 RUN pip3 install -r requirements.txt --upgrade
-RUN pip3 install /root/dist/* --upgrade
+#RUN pip3 install /root/dist/* --upgrade
 RUN pip3 install .
 
 CMD ["python3", "-m", "moon_interface"]
\ No newline at end of file
diff --git a/moonv4/moon_interface/moon_interface/authz_requests.py b/moonv4/moon_interface/moon_interface/authz_requests.py
index 10356787..b6b0e58e 100644
--- a/moonv4/moon_interface/moon_interface/authz_requests.py
+++ b/moonv4/moon_interface/moon_interface/authz_requests.py
@@ -28,8 +28,6 @@ class AuthzRequest:
         self.context = Context(ctx, CACHE)
         self.args = args
         self.request_id = ctx["request_id"]
-        # LOG.info("container={}".format(CACHE.containers))
-        # LOG.info("container_chaining={}".format(CACHE.container_chaining))
         if ctx['project_id'] not in CACHE.container_chaining:
             raise exceptions.KeystoneProjectError("Unknown Project ID {}".format(ctx['project_id']))
         self.container_chaining = CACHE.container_chaining[ctx['project_id']]
@@ -40,36 +38,35 @@ class AuthzRequest:
 
     def run(self):
         self.context.delete_cache()
+        req = None
         try:
-            LOG.debug("url=http://{}:{}/authz".format(
-                self.container_chaining[0]["hostname"],
-                self.container_chaining[0]["port"]))
             req = requests.post("http://{}:{}/authz".format(
-                self.container_chaining[0]["hostname"],
+                self.container_chaining[0]["hostip"],
                 self.container_chaining[0]["port"],
             ), data=pickle.dumps(self.context))
             if req.status_code != 200:
-                # LOG.error("Cannot connect to {}".format(
-                #     "http://{}:{}/authz".format(
-                #         self.container_chaining[0]["hostname"],
-                #         self.container_chaining[0]["port"]
-                #     )))
+                # LOG.error("req={}".format(req))
                 raise exceptions.AuthzException(
                     "Receive bad response from Authz function "
-                    "(with hostname - {})".format(
+                    "(with IP address - {})".format(
                         req.status_code
                     ))
         except requests.exceptions.ConnectionError:
+            LOG.error("Cannot connect to {}".format(
+                "http://{}:{}/authz".format(
+                    self.container_chaining[0]["hostip"],
+                    self.container_chaining[0]["port"]
+                )))
+        except ValueError:
             try:
                 req = requests.post("http://{}:{}/authz".format(
-                    self.container_chaining[0]["hostip"],
+                    self.container_chaining[0]["hostname"],
                     self.container_chaining[0]["port"],
                 ), data=pickle.dumps(self.context))
                 if req.status_code != 200:
-                    # LOG.error("req={}".format(req))
                     raise exceptions.AuthzException(
                         "Receive bad response from Authz function "
-                        "(with IP address - {})".format(
+                        "(with hostname - {})".format(
                             req.status_code
                         ))
             except requests.exceptions.ConnectionError:
@@ -81,7 +78,7 @@ class AuthzRequest:
                 raise exceptions.AuthzException(
                     "Cannot connect to Authz function with IP address")
         self.context.set_cache(CACHE)
-        if len(self.container_chaining) == 1:
+        if req and len(self.container_chaining) == 1:
             # req.raw.decode_content = True
             self.result = pickle.loads(req.content)
 
diff --git a/moonv4/moon_manager/Dockerfile b/moonv4/moon_manager/Dockerfile
index 17bee16a..873e3aa2 100644
--- a/moonv4/moon_manager/Dockerfile
+++ b/moonv4/moon_manager/Dockerfile
@@ -6,7 +6,7 @@ RUN pip3 install pip --upgrade
 ADD . /root
 WORKDIR /root/
 RUN pip3 install -r requirements.txt
-RUN pip3 install /root/dist/* --upgrade
+#RUN pip3 install /root/dist/* --upgrade
 RUN pip3 install .
 
 CMD ["python3", "-m", "moon_manager"]
\ No newline at end of file
diff --git a/moonv4/moon_manager/moon_manager/api/pdp.py b/moonv4/moon_manager/moon_manager/api/pdp.py
index ffc91fba..465f4dba 100644
--- a/moonv4/moon_manager/moon_manager/api/pdp.py
+++ b/moonv4/moon_manager/moon_manager/api/pdp.py
@@ -14,6 +14,8 @@ import requests
 import time
 from moon_utilities.security_functions import check_auth
 from moon_db.core import PDPManager
+from moon_db.core import PolicyManager
+from moon_db.core import ModelManager
 from moon_utilities import configuration
 
 __version__ = "0.1.0"
@@ -26,15 +28,23 @@ def delete_pod(uuid):
 
 
 def add_pod(uuid, data):
+    if not data.get("keystone_project_id"):
+        return
     LOG.info("Add a new pod {}".format(data))
+    if "pdp_id" not in data:
+        data["pdp_id"] = uuid
+    data['policies'] = PolicyManager.get_policies(user_id="admin")
+    data['models'] = ModelManager.get_models(user_id="admin")
     conf = configuration.get_configuration("components/orchestrator")
     hostname = conf["components/orchestrator"].get("hostname", "orchestrator")
     port = conf["components/orchestrator"].get("port", 80)
     proto = conf["components/orchestrator"].get("protocol", "http")
     while True:
         try:
-            req = requests.post("{}://{}:{}/pods".format(proto, hostname, port),
-                                data=data)
+            req = requests.post(
+                "{}://{}:{}/pods".format(proto, hostname, port),
+                json=data,
+                headers={"content-type": "application/json"})
         except requests.exceptions.ConnectionError:
             LOG.warning("Orchestrator is not ready, standby...")
             time.sleep(1)
diff --git a/moonv4/moon_orchestrator/moon_orchestrator/api/pods.py b/moonv4/moon_orchestrator/moon_orchestrator/api/pods.py
index a7ca1cbc..c5c5b2c4 100644
--- a/moonv4/moon_orchestrator/moon_orchestrator/api/pods.py
+++ b/moonv4/moon_orchestrator/moon_orchestrator/api/pods.py
@@ -25,6 +25,7 @@ class Pods(Resource):
 
     def __init__(self, **kwargs):
         self.driver = kwargs.get("driver")
+        self.create_security_function = kwargs.get("create_security_function_hook")
 
     @check_auth
     def get(self, uuid=None, user_id=None):
@@ -72,7 +73,15 @@ class Pods(Resource):
             }
         }
         """
-        return {"pods": None}
+        LOG.info("POST param={}".format(request.json))
+        self.create_security_function(
+            request.json.get("keystone_project_id"),
+            request.json.get("pdp_id"),
+            request.json.get("security_pipeline"),
+            manager_data=request.json,
+            active_context=None,
+            active_context_name=None)
+        return {"pods": self.driver.get_pods(request.json.get("pdp_id"))}
 
     @check_auth
     def delete(self, uuid=None, user_id=None):
diff --git a/moonv4/moon_orchestrator/moon_orchestrator/drivers.py b/moonv4/moon_orchestrator/moon_orchestrator/drivers.py
index 63631c00..63ca8f3c 100644
--- a/moonv4/moon_orchestrator/moon_orchestrator/drivers.py
+++ b/moonv4/moon_orchestrator/moon_orchestrator/drivers.py
@@ -8,14 +8,14 @@ import logging
 import urllib3.exceptions
 from moon_utilities import configuration
 
-logger = logging.getLogger("moon.orchestrator.drivers")
+LOG = logging.getLogger("moon.orchestrator.drivers")
 
 
 def get_driver():
     try:
         return K8S()
     except urllib3.exceptions.MaxRetryError as e:
-        logger.exception(e)
+        LOG.exception(e)
         return Docker()
 
 
@@ -60,12 +60,12 @@ class K8S(Driver):
         if name:
             pods = self.client.list_pod_for_all_namespaces(watch=False)
             for pod in pods.items:
-                logger.info("get_pods {}".format(pod.metadata.name))
+                LOG.info("get_pods {}".format(pod.metadata.name))
                 if name in pod.metadata.name:
                     return pod
             else:
                 return None
-        logger.info("get_pods cache={}".format(self.cache))
+        LOG.info("get_pods cache={}".format(self.cache))
         return self.cache
 
     @staticmethod
@@ -101,7 +101,7 @@ class K8S(Driver):
                         {'name': "TYPE", "value": _data.get('genre', "None")},
                         {'name': "PORT", "value": str(_data.get('port', 80))},
                         {'name': "PDP_ID", "value": _data.get('pdp_id', "None")},
-                        {'name': "META_RULE_ID", "value": "None"},
+                        {'name': "META_RULE_ID", "value": _data.get('meta_rule_id', "None")},
                         {'name': "KEYSTONE_PROJECT_ID",
                          "value": _data.get('keystone_project_id', "None")},
                     ]
@@ -109,7 +109,7 @@ class K8S(Driver):
             )
         resp = client.create_namespaced_deployment(body=pod_manifest,
                                                    namespace='moon')
-        logger.info("Pod {} created!".format(data[0].get('name')))
+        LOG.info("Pod {} created!".format(data[0].get('name')))
         # logger.info(yaml.dump(pod_manifest, sys.stdout))
         # logger.info(resp)
         return resp
@@ -131,7 +131,7 @@ class K8S(Driver):
                 'selector': {
                     'app': data.get('name')
                 },
-                'type': 'NodePort',
+                # 'type': 'NodePort',
                 'endpoints': [{
                     'port': data.get('port', 80),
                     'protocol': 'TCP',
@@ -144,7 +144,7 @@ class K8S(Driver):
             service_manifest['spec']['type'] = "NodePort"
         resp = client.create_namespaced_service(namespace="moon",
                                                 body=service_manifest)
-        logger.info("Service {} created!".format(data.get('name')))
+        LOG.info("Service {} created!".format(data.get('name')))
         return resp
 
     def load_pod(self, data, api_client=None, ext_client=None, expose=False):
@@ -152,12 +152,12 @@ class K8S(Driver):
         pod = self.__create_pod(client=ext_client, data=data)
         service = self.__create_service(client=_client, data=data[0],
                                         expose=expose)
-        # logger.info("load_poad data={}".format(data))
+        # logger.info("load_pod data={}".format(data))
         # logger.info("pod.metadata.uid={}".format(pod.metadata.uid))
         self.cache[pod.metadata.uid] = data
 
     def delete_pod(self, uuid=None, name=None):
-        logger.info("Deleting pod {}".format(uuid))
+        LOG.info("Deleting pod {}".format(uuid))
         # TODO: delete_namespaced_deployment
         # https://github.com/kubernetes-incubator/client-python/blob/master/kubernetes/client/apis/extensions_v1beta1_api.py
 
@@ -169,9 +169,9 @@ class K8S(Driver):
 class Docker(Driver):
 
     def load_pod(self, data, api_client=None, ext_client=None):
-        logger.info("Creating pod {}".format(data[0].get('name')))
+        LOG.info("Creating pod {}".format(data[0].get('name')))
         raise NotImplementedError
 
     def delete_pod(self, uuid=None, name=None):
-        logger.info("Deleting pod {}".format(uuid))
+        LOG.info("Deleting pod {}".format(uuid))
         raise NotImplementedError
diff --git a/moonv4/moon_orchestrator/moon_orchestrator/http_server.py b/moonv4/moon_orchestrator/moon_orchestrator/http_server.py
index 70c5e2d2..c9816f5b 100644
--- a/moonv4/moon_orchestrator/moon_orchestrator/http_server.py
+++ b/moonv4/moon_orchestrator/moon_orchestrator/http_server.py
@@ -154,6 +154,8 @@ class HTTPServer(Server):
         self.api.add_resource(Pods, *Pods.__urls__,
                               resource_class_kwargs={
                                   "driver": self.driver,
+                                  "create_security_function_hook":
+                                      self.create_security_function,
                               })
 
     def run(self):
@@ -190,7 +192,8 @@ class HTTPServer(Server):
             LOG.debug('wrapper pod={}'.format(pod))
 
     def create_security_function(self, keystone_project_id,
-                                 pdp_id, policy_ids, active_context=None,
+                                 pdp_id, policy_ids, manager_data={},
+                                 active_context=None,
                                  active_context_name=None):
         """ Create security functions
 
@@ -203,13 +206,14 @@ class HTTPServer(Server):
         security function in all context (ie, in all slaves)
         :return: None
         """
-        LOG.debug(self.driver.get_pods())
+        # LOG.info(self.driver.get_pods())
         for key, value in self.driver.get_pods().items():
             for _pod in value:
                 if _pod.get('keystone_project_id') == keystone_project_id:
                     LOG.warning("A pod for this Keystone project {} "
                                    "already exists.".format(keystone_project_id))
                     return
+
         plugins = configuration.get_plugins()
         conf = configuration.get_configuration("components/interface")
         i_hostname = conf["components/interface"].get("hostname", "interface")
@@ -228,12 +232,21 @@ class HTTPServer(Server):
                 "namespace": "moon"
             },
         ]
-        policies = requests.get("http://{}:{}/policies".format(
-            self.manager_hostname, self.manager_port)).json().get(
-            "policies", dict())
-        models = requests.get("http://{}:{}/models".format(
-            self.manager_hostname, self.manager_port)).json().get(
-            "models", dict())
+        LOG.info("data={}".format(data))
+        policies = manager_data.get('policies')
+        if not policies:
+            LOG.info("No policy data from Manager, trying to get them")
+            policies = requests.get("http://{}:{}/policies".format(
+                self.manager_hostname, self.manager_port)).json().get(
+                "policies", dict())
+        LOG.info("policies={}".format(policies))
+        models = manager_data.get('models')
+        if not models:
+            LOG.info("No models data from Manager, trying to get them")
+            models = requests.get("http://{}:{}/models".format(
+                self.manager_hostname, self.manager_port)).json().get(
+                "models", dict())
+        LOG.info("models={}".format(models))
 
         for policy_id in policy_ids:
             if policy_id in policies:
@@ -251,7 +264,10 @@ class HTTPServer(Server):
                             'keystone_project_id': keystone_project_id,
                             "namespace": "moon"
                         })
+        LOG.info("data={}".format(data))
         contexts, _active_context = self.driver.get_slaves()
+        LOG.info("active_context_name={}".format(active_context_name))
+        LOG.info("active_context={}".format(active_context))
         if active_context_name:
             for _context in contexts:
                 if _context["name"] == active_context_name:
@@ -264,13 +280,14 @@ class HTTPServer(Server):
             LOG.debug("_config={}".format(_config))
             api_client = client.CoreV1Api(_config)
             ext_client = client.ExtensionsV1beta1Api(_config)
-            self.driver.load_pod(data, api_client, ext_client)
+            self.driver.load_pod(data, api_client, ext_client, expose=False)
             return
+        LOG.info("contexts={}".format(contexts))
         for _ctx in contexts:
             _config = config.new_client_from_config(context=_ctx['name'])
             LOG.debug("_config={}".format(_config))
             api_client = client.CoreV1Api(_config)
             ext_client = client.ExtensionsV1beta1Api(_config)
-            self.driver.load_pod(data, api_client, ext_client)
+            self.driver.load_pod(data, api_client, ext_client, expose=False)
 
 
diff --git a/moonv4/moon_utilities/Changelog b/moonv4/moon_utilities/Changelog
index 185e81dc..ca67cbd4 100644
--- a/moonv4/moon_utilities/Changelog
+++ b/moonv4/moon_utilities/Changelog
@@ -51,3 +51,7 @@ CHANGES
 -----
 - Update the cache
 
+1.3.4
+-----
+- Fix a bug on the connection between interface and authz
+
diff --git a/moonv4/moon_utilities/build.sh b/moonv4/moon_utilities/build.sh
index 79bd9132..b327e49d 100644
--- a/moonv4/moon_utilities/build.sh
+++ b/moonv4/moon_utilities/build.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-VERSION=moon_utilities-1.3.3
+VERSION=moon_utilities-1.3.4
 
 python3 setup.py sdist bdist_wheel
 
diff --git a/moonv4/moon_utilities/moon_utilities/__init__.py b/moonv4/moon_utilities/moon_utilities/__init__.py
index 327860b3..3e59b6a0 100644
--- a/moonv4/moon_utilities/moon_utilities/__init__.py
+++ b/moonv4/moon_utilities/moon_utilities/__init__.py
@@ -3,4 +3,4 @@
 # license which can be found in the file 'LICENSE' in this package distribution
 # or at 'http://www.apache.org/licenses/LICENSE-2.0'.
 
-__version__ = "1.3.3"
+__version__ = "1.3.4"
diff --git a/moonv4/moon_utilities/moon_utilities/cache.py b/moonv4/moon_utilities/moon_utilities/cache.py
index e1d2477a..d62c9d54 100644
--- a/moonv4/moon_utilities/moon_utilities/cache.py
+++ b/moonv4/moon_utilities/moon_utilities/cache.py
@@ -520,6 +520,7 @@ class Cache(object):
                                         "policy_id": policy_id,
                                         "meta_rule_id": meta_rule_id,
                                         "hostname": container_value["name"],
+                                        "hostip": "127.0.0.1",
                                         "port": container_value["port"],
                                     }
                                 )
diff --git a/moonv4/moon_wrapper/Dockerfile b/moonv4/moon_wrapper/Dockerfile
index 958f136c..55e7208d 100644
--- a/moonv4/moon_wrapper/Dockerfile
+++ b/moonv4/moon_wrapper/Dockerfile
@@ -6,7 +6,7 @@ RUN pip3 install pip --upgrade
 ADD . /root
 WORKDIR /root/
 RUN pip3 install -r requirements.txt --upgrade
-#RUN pip3 install /root/dist/* --upgrade
+RUN pip3 install /root/dist/* --upgrade
 RUN pip3 install .
 
 CMD ["python3", "-m", "moon_wrapper"]
diff --git a/moonv4/templates/moonforming/moon.conf b/moonv4/templates/moonforming/moon.conf
index 0d3f740f..2fa1db55 100644
--- a/moonv4/templates/moonforming/moon.conf
+++ b/moonv4/templates/moonforming/moon.conf
@@ -29,12 +29,14 @@ openstack:
 plugins:
     authz:
         container: wukongsun/moon_authz:v4.3
+        port: 8081
     session:
         container: asteroide/session:latest
+        port: 8082
 
 components:
     interface:
-        port: 8081
+        port: 8080
         bind: 0.0.0.0
         hostname: interface
         container: wukongsun/moon_interface:v4.3