Implement the delete_pod function in Manager and Orchestrator

Change-Id: I55da3610fa791ec9b3343cca49c5ca034cc7579f

5 files changed, 273 insertions, 187 deletions

diff --git a/moon_orchestrator/moon_orchestrator/api/pods.py b/moon_orchestrator/moon_orchestrator/api/pods.py
index 7b89372e..a83ca9ae 100644
--- a/moon_orchestrator/moon_orchestrator/api/pods.py
+++ b/moon_orchestrator/moon_orchestrator/api/pods.py
@@ -27,7 +27,6 @@ class Pods(Resource):
 
     def __init__(self, **kwargs):
         self.driver = kwargs.get("driver")
-        self.create_pipeline = kwargs.get("create_pipeline_hook")
 
     @check_auth
     def get(self, uuid=None, user_id=None):
@@ -75,7 +74,7 @@ class Pods(Resource):
         }
         """
         logger.debug("POST param={}".format(request.json))
-        self.create_pipeline(
+        self.driver.create_pipeline(
             request.json.get("keystone_project_id"),
             request.json.get("pdp_id"),
             request.json.get("security_pipeline"),
@@ -102,7 +101,11 @@ class Pods(Resource):
             "message": "optional message"
         }
         """
-        return {"result": True}
+        try:
+            self.driver.delete_pipeline(uuid)
+            return {'result': True}
+        except Exception as e:
+            return {"result": False, "message": str(e)}, 500
 
     @check_auth
     def patch(self, uuid=None, user_id=None):
diff --git a/moon_orchestrator/moon_orchestrator/drivers.py b/moon_orchestrator/moon_orchestrator/drivers.py
index 07f012aa..b21f2639 100644
--- a/moon_orchestrator/moon_orchestrator/drivers.py
+++ b/moon_orchestrator/moon_orchestrator/drivers.py
@@ -5,8 +5,10 @@
 
 from kubernetes import client, config
 import logging
+import requests
 import urllib3.exceptions
-from python_moonutilities import configuration
+from python_moonutilities import configuration, exceptions
+from python_moonutilities.misc import get_random_name
 
 logger = logging.getLogger("moon.orchestrator.drivers")
 
@@ -36,16 +38,24 @@ class Driver:
         #     }
         # }
 
-    def get_pods(self, namespace=None):
+    def get_slaves(self):
         raise NotImplementedError
 
-    def load_pod(self, data, api_client=None, ext_client=None):
+    def create_wrappers(self):
         raise NotImplementedError
 
-    def delete_pod(self, uuid=None, name=None):
+    def delete_wrapper(self, name):
         raise NotImplementedError
 
-    def get_slaves(self):
+    def create_pipeline(self, keystone_project_id,
+                        pdp_id, policy_ids, manager_data=None,
+                        active_context=None,
+                        active_context_name=None):
+        raise NotImplementedError
+
+    def delete_pipeline(self, uuid=None, name=None, namespace="moon",
+                        active_context=None,
+                        active_context_name=None):
         raise NotImplementedError
 
 
@@ -55,6 +65,12 @@ class K8S(Driver):
         super(K8S, self).__init__()
         config.load_kube_config()
         self.client = client.CoreV1Api()
+        conf = configuration.get_configuration("components/orchestrator")
+        self.orchestrator_hostname = conf["components/orchestrator"].get("hostname", "orchestrator")
+        self.orchestrator_port = conf["components/orchestrator"].get("port", 80)
+        conf = configuration.get_configuration("components/manager")
+        self.manager_hostname = conf["components/manager"].get("hostname", "manager")
+        self.manager_port = conf["components/manager"].get("port", 80)
 
     def get_pods(self, name=None):
         if name:
@@ -69,7 +85,7 @@ class K8S(Driver):
         return self.cache
 
     @staticmethod
-    def __create_pod(client, data):
+    def __create_deployment(client, data):
         pod_manifest = {
             'apiVersion': 'extensions/v1beta1',
             'kind': 'Deployment',
@@ -145,29 +161,231 @@ class K8S(Driver):
         logger.info("Service {} created!".format(data.get('name')))
         return resp
 
-    def load_pod(self, data, api_client=None, ext_client=None, expose=False):
+    def load_deployment_and_service(self, data, api_client=None, ext_client=None, expose=False):
         _client = api_client if api_client else self.client
-        pod = self.__create_pod(client=ext_client, data=data)
+        pod = self.__create_deployment(client=ext_client, data=data)
         self.__create_service(client=_client, data=data[0],
-                                        expose=expose)
+                              expose=expose)
         self.cache[pod.metadata.uid] = data
 
-    def delete_pod(self, uuid=None, name=None):
-        logger.info("Deleting pod {}".format(uuid))
-        # TODO: delete_namespaced_deployment
-        # https://github.com/kubernetes-incubator/client-python/blob/master/kubernetes/client/apis/extensions_v1beta1_api.py
+    @staticmethod
+    def delete_deployment(name=None, namespace="moon", ext_client=None):
+        logger.info("Deleting deployment {}".format(name))
+        body = client.V1DeleteOptions(propagation_policy='Foreground')
+        ret = ext_client.delete_namespaced_deployment(
+            name=name,
+            namespace=namespace,
+            body=body
+        )
+        logger.info(ret)
+
+    def delete_service(self, name, namespace="moon", api_client=None):
+        if not api_client:
+            api_client = self.client
+        ret = api_client.delete_namespaced_service(name=name, namespace=namespace)
+        logger.debug("delete_service {}".format(ret))
 
     def get_slaves(self):
         contexts, active_context = config.list_kube_config_contexts()
         return contexts, active_context
 
+    def create_wrappers(self):
+        contexts, active_context = self.get_slaves()
+        logger.debug("contexts: {}".format(contexts))
+        logger.debug("active_context: {}".format(active_context))
+        conf = configuration.get_configuration("components/wrapper")
+        hostname = conf["components/wrapper"].get(
+            "hostname", "wrapper")
+        port = conf["components/wrapper"].get("port", 80)
+        container = conf["components/wrapper"].get(
+            "container",
+            "wukongsun/moon_wrapper:v4.3")
+        for _ctx in contexts:
+            _config = config.new_client_from_config(context=_ctx['name'])
+            logger.debug("_config={}".format(_config))
+            api_client = client.CoreV1Api(_config)
+            ext_client = client.ExtensionsV1beta1Api(_config)
+            data = [{
+                "name": hostname + "-" + get_random_name(),
+                "container": container,
+                "port": port,
+                "namespace": "moon"
+            }, ]
+            self.load_deployment_and_service(data, api_client, ext_client, expose=True)
+
+    def create_pipeline(self, keystone_project_id,
+                        pdp_id, policy_ids, manager_data=None,
+                        active_context=None,
+                        active_context_name=None):
+        """ Create security functions
+
+        :param keystone_project_id: the Keystone project id
+        :param pdp_id: the PDP ID mapped to this pipeline
+        :param policy_ids: the policy IDs mapped to this pipeline
+        :param manager_data: data needed to create pods
+        :param active_context: if present, add the security function in this
+                               context
+        :param active_context_name: if present,  add the security function in
+                                    this context name
+        if active_context_name and active_context are not present, add the
+        security function in all context (ie, in all slaves)
+        :return: None
+        """
+        if not manager_data:
+            manager_data = dict()
+        for key, value in self.get_pods().items():
+            for _pod in value:
+                if _pod.get('keystone_project_id') == keystone_project_id:
+                    logger.warning("A pod for this Keystone project {} "
+                                   "already exists.".format(keystone_project_id))
+                    return
+
+        plugins = configuration.get_plugins()
+        conf = configuration.get_configuration("components/pipeline")
+        # i_hostname = conf["components/pipeline"].get("interface").get("hostname", "interface")
+        i_port = conf["components/pipeline"].get("interface").get("port", 80)
+        i_container = conf["components/pipeline"].get("interface").get(
+            "container",
+            "wukongsun/moon_interface:v4.3")
+        data = [
+            {
+                "name": "pipeline-" + get_random_name(),
+                "container": i_container,
+                "port": i_port,
+                'pdp_id': pdp_id,
+                'genre': "interface",
+                'keystone_project_id': keystone_project_id,
+                "namespace": "moon"
+            },
+        ]
+        logger.debug("data={}".format(data))
+        policies = manager_data.get('policies')
+        if not policies:
+            logger.info("No policy data from Manager, trying to get them")
+            policies = requests.get("http://{}:{}/policies".format(
+                self.manager_hostname, self.manager_port)).json().get(
+                "policies", dict())
+        logger.debug("policies={}".format(policies))
+        models = manager_data.get('models')
+        if not models:
+            logger.info("No models data from Manager, trying to get them")
+            models = requests.get("http://{}:{}/models".format(
+                self.manager_hostname, self.manager_port)).json().get(
+                "models", dict())
+        logger.debug("models={}".format(models))
+
+        for policy_id in policy_ids:
+            if policy_id in policies:
+                genre = policies[policy_id].get("genre", "authz")
+                if genre in plugins:
+                    for meta_rule in models[policies[policy_id]['model_id']]['meta_rules']:
+                        data.append({
+                            "name": genre + "-" + get_random_name(),
+                            "container": plugins[genre]['container'],
+                            'pdp_id': pdp_id,
+                            "port": plugins[genre].get('port', 8080),
+                            'genre': genre,
+                            'policy_id': policy_id,
+                            'meta_rule_id': meta_rule,
+                            'keystone_project_id': keystone_project_id,
+                            "namespace": "moon"
+                        })
+        logger.debug("data={}".format(data))
+        contexts, _active_context = self.get_slaves()
+        logger.debug("active_context_name={}".format(active_context_name))
+        logger.debug("active_context={}".format(active_context))
+        if active_context_name:
+            for _context in contexts:
+                if _context["name"] == active_context_name:
+                    active_context = _context
+                    break
+        if active_context:
+            active_context = _active_context
+            _config = config.new_client_from_config(
+                context=active_context['name'])
+            logger.debug("_config={}".format(_config))
+            api_client = client.CoreV1Api(_config)
+            ext_client = client.ExtensionsV1beta1Api(_config)
+            self.load_deployment_and_service(data, api_client, ext_client, expose=False)
+            return
+        logger.debug("contexts={}".format(contexts))
+        for _ctx in contexts:
+            _config = config.new_client_from_config(context=_ctx['name'])
+            logger.debug("_config={}".format(_config))
+            api_client = client.CoreV1Api(_config)
+            ext_client = client.ExtensionsV1beta1Api(_config)
+            self.load_deployment_and_service(data, api_client, ext_client, expose=False)
+
+    def delete_pipeline(self, uuid=None, name=None, namespace="moon",
+                        active_context=None,
+                        active_context_name=None):
+        """Delete a pipeline
+
+        :param uuid:
+        :param name:
+        :param namespace:
+        :param active_context:
+        :param active_context_name:
+        :return:
+        """
+        name_to_delete = None
+        if uuid and uuid in self.get_pods():
+            name_to_delete = self.get_pods()[uuid][0]['name']
+        elif name:
+            for pod_key, pod_list in self.get_pods().items():
+                for pod_value in pod_list:
+                    if pod_value.get("name") == name:
+                        name_to_delete = pod_value.get("name")
+                        break
+        if not name_to_delete:
+            raise exceptions.MoonError("Cannot find pipeline")
+        logger.info("Will delete deployment and service named {}".format(name_to_delete))
+        contexts, _active_context = self.get_slaves()
+        if active_context_name:
+            for _context in contexts:
+                if _context["name"] == active_context_name:
+                    active_context = _context
+                    break
+        if active_context:
+            active_context = _active_context
+            _config = config.new_client_from_config(
+                context=active_context['name'])
+            logger.debug("_config={}".format(_config))
+            api_client = client.CoreV1Api(_config)
+            ext_client = client.ExtensionsV1beta1Api(_config)
+            self.delete_deployment(name=name_to_delete, namespace=namespace,
+                                   ext_client=ext_client)
+            self.delete_service(name=name_to_delete, api_client=api_client)
+            return
+        logger.debug("contexts={}".format(contexts))
+        for _ctx in contexts:
+            _config = config.new_client_from_config(context=_ctx['name'])
+            logger.debug("_config={}".format(_config))
+            api_client = client.CoreV1Api(_config)
+            ext_client = client.ExtensionsV1beta1Api(_config)
+            self.delete_deployment(name=name_to_delete, namespace=namespace,
+                                   ext_client=ext_client)
+            self.delete_service(name=name_to_delete, api_client=api_client)
+
 
 class Docker(Driver):
 
-    def load_pod(self, data, api_client=None, ext_client=None):
-        logger.info("Creating pod {}".format(data[0].get('name')))
+    def get_slaves(self):
+        raise NotImplementedError
+
+    def create_wrappers(self):
+        raise NotImplementedError
+
+    def delete_wrapper(self, name):
+        raise NotImplementedError
+
+    def create_pipeline(self, keystone_project_id,
+                        pdp_id, policy_ids, manager_data=None,
+                        active_context=None,
+                        active_context_name=None):
         raise NotImplementedError
 
-    def delete_pod(self, uuid=None, name=None):
-        logger.info("Deleting pod {}".format(uuid))
+    def delete_pipeline(self, uuid=None, name=None, namespace="moon",
+                        active_context=None,
+                        active_context_name=None):
         raise NotImplementedError
diff --git a/moon_orchestrator/moon_orchestrator/http_server.py b/moon_orchestrator/moon_orchestrator/http_server.py
index 00be0335..fa5308d0 100644
--- a/moon_orchestrator/moon_orchestrator/http_server.py
+++ b/moon_orchestrator/moon_orchestrator/http_server.py
@@ -5,7 +5,6 @@
 
 from flask import Flask, jsonify
 from flask_restful import Resource, Api
-from kubernetes import client, config
 import logging
 import requests
 import time
@@ -14,7 +13,6 @@ from moon_orchestrator.api.pods import Pods
 from moon_orchestrator.api.generic import Status
 from moon_orchestrator.drivers import get_driver
 from python_moonutilities import configuration, exceptions
-from python_moonutilities.misc import get_random_name
 
 logger = logging.getLogger("moon.orchestrator.http_server")
 
@@ -124,11 +122,11 @@ class HTTPServer(Server):
                 if "pdps" in pdp.json():
                     break
         logger.debug("pdp={}".format(pdp))
-        self.create_wrappers()
+        self.driver.create_wrappers()
         for _pdp_key, _pdp_value in pdp.json()['pdps'].items():
             if _pdp_value.get('keystone_project_id'):
                 # TODO: select context to add security function
-                self.create_pipeline(
+                self.driver.create_pipeline(
                     keystone_project_id=_pdp_value.get('keystone_project_id'),
                     pdp_id=_pdp_key,
                     policy_ids=_pdp_value.get('security_pipeline', []))
@@ -151,9 +149,7 @@ class HTTPServer(Server):
             self.api.add_resource(api, *api.__urls__)
         self.api.add_resource(Pods, *Pods.__urls__,
                               resource_class_kwargs={
-                                  "driver": self.driver,
-                                  "create_pipeline_hook":
-                                      self.create_pipeline,
+                                  "driver": self.driver
                               })
 
     def run(self):
@@ -163,132 +159,3 @@ class HTTPServer(Server):
     def __filter_str(data):
         return data.replace("@", "-")
 
-    def create_wrappers(self):
-        contexts, active_context = self.driver.get_slaves()
-        logger.debug("contexts: {}".format(contexts))
-        logger.debug("active_context: {}".format(active_context))
-        conf = configuration.get_configuration("components/wrapper")
-        hostname = conf["components/wrapper"].get(
-            "hostname", "wrapper")
-        port = conf["components/wrapper"].get("port", 80)
-        container = conf["components/wrapper"].get(
-            "container",
-            "wukongsun/moon_wrapper:v4.3")
-        for _ctx in contexts:
-            _config = config.new_client_from_config(context=_ctx['name'])
-            logger.debug("_config={}".format(_config))
-            api_client = client.CoreV1Api(_config)
-            ext_client = client.ExtensionsV1beta1Api(_config)
-            data = [{
-                "name": hostname + "-" + get_random_name(),
-                "container": container,
-                "port": port,
-                "namespace": "moon"
-            }, ]
-            pod = self.driver.load_pod(data, api_client, ext_client, expose=True)
-            logger.debug('wrapper pod={}'.format(pod))
-
-    def create_pipeline(self, keystone_project_id,
-                        pdp_id, policy_ids, manager_data=None,
-                        active_context=None,
-                        active_context_name=None):
-        """ Create security functions
-
-        :param keystone_project_id: the Keystone project id
-        :param pdp_id: the PDP ID mapped to this pipeline
-        :param policy_ids: the policy IDs mapped to this pipeline
-        :param manager_data: data needed to create pods
-        :param active_context: if present, add the security function in this
-                               context
-        :param active_context_name: if present,  add the security function in
-                                    this context name
-        if active_context_name and active_context are not present, add the
-        security function in all context (ie, in all slaves)
-        :return: None
-        """
-        if not manager_data:
-            manager_data = dict()
-        for key, value in self.driver.get_pods().items():
-            for _pod in value:
-                if _pod.get('keystone_project_id') == keystone_project_id:
-                    logger.warning("A pod for this Keystone project {} "
-                                   "already exists.".format(keystone_project_id))
-                    return
-
-        plugins = configuration.get_plugins()
-        conf = configuration.get_configuration("components/pipeline")
-        # i_hostname = conf["components/pipeline"].get("interface").get("hostname", "interface")
-        i_port = conf["components/pipeline"].get("interface").get("port", 80)
-        i_container = conf["components/pipeline"].get("interface").get(
-            "container",
-            "wukongsun/moon_interface:v4.3")
-        data = [
-            {
-                "name": "pipeline-" + get_random_name(),
-                "container": i_container,
-                "port": i_port,
-                'pdp_id': pdp_id,
-                'genre': "interface",
-                'keystone_project_id': keystone_project_id,
-                "namespace": "moon"
-            },
-        ]
-        logger.debug("data={}".format(data))
-        policies = manager_data.get('policies')
-        if not policies:
-            logger.info("No policy data from Manager, trying to get them")
-            policies = requests.get("http://{}:{}/policies".format(
-                self.manager_hostname, self.manager_port)).json().get(
-                "policies", dict())
-        logger.debug("policies={}".format(policies))
-        models = manager_data.get('models')
-        if not models:
-            logger.info("No models data from Manager, trying to get them")
-            models = requests.get("http://{}:{}/models".format(
-                self.manager_hostname, self.manager_port)).json().get(
-                "models", dict())
-        logger.debug("models={}".format(models))
-
-        for policy_id in policy_ids:
-            if policy_id in policies:
-                genre = policies[policy_id].get("genre", "authz")
-                if genre in plugins:
-                    for meta_rule in models[policies[policy_id]['model_id']]['meta_rules']:
-                        data.append({
-                            "name": genre + "-" + get_random_name(),
-                            "container": plugins[genre]['container'],
-                            'pdp_id': pdp_id,
-                            "port": plugins[genre].get('port', 8080),
-                            'genre': genre,
-                            'policy_id': policy_id,
-                            'meta_rule_id': meta_rule,
-                            'keystone_project_id': keystone_project_id,
-                            "namespace": "moon"
-                        })
-        logger.debug("data={}".format(data))
-        contexts, _active_context = self.driver.get_slaves()
-        logger.debug("active_context_name={}".format(active_context_name))
-        logger.debug("active_context={}".format(active_context))
-        if active_context_name:
-            for _context in contexts:
-                if _context["name"] == active_context_name:
-                    active_context = _context
-                    break
-        if active_context:
-            active_context = _active_context
-            _config = config.new_client_from_config(
-                context=active_context['name'])
-            logger.debug("_config={}".format(_config))
-            api_client = client.CoreV1Api(_config)
-            ext_client = client.ExtensionsV1beta1Api(_config)
-            self.driver.load_pod(data, api_client, ext_client, expose=False)
-            return
-        logger.debug("contexts={}".format(contexts))
-        for _ctx in contexts:
-            _config = config.new_client_from_config(context=_ctx['name'])
-            logger.debug("_config={}".format(_config))
-            api_client = client.CoreV1Api(_config)
-            ext_client = client.ExtensionsV1beta1Api(_config)
-            self.driver.load_pod(data, api_client, ext_client, expose=False)
-
-
diff --git a/moon_orchestrator/tests/unit_python/mock_pods.py b/moon_orchestrator/tests/unit_python/mock_pods.py
index c5633152..84e6c7ea 100644
--- a/moon_orchestrator/tests/unit_python/mock_pods.py
+++ b/moon_orchestrator/tests/unit_python/mock_pods.py
@@ -251,11 +251,11 @@ def register_consul(m):
             json=[{'Key': component, 'Value': get_b64_conf(component)}]
         )
     m.register_uri(
-        'GET', 'http://consul:8500/v1/kv/components_port_start',
+        'GET', 'http://consul:8500/v1/kv/components/port_start',
         json=[
             {
                 "LockIndex": 0,
-                "Key": "components_port_start",
+                "Key": "components/port_start",
                 "Flags": 0,
                 "Value": "MzEwMDE=",
                 "CreateIndex": 9,
@@ -264,22 +264,22 @@ def register_consul(m):
         ],
     )
     m.register_uri(
-        'PUT', 'http://consul:8500/v1/kv/components_port_start',
+        'PUT', 'http://consul:8500/v1/kv/components/port_start',
         json=[],
     )
-    m.register_uri(
-        'GET', 'http://consul:8500/v1/kv/plugins?recurse=true',
-        json=[
-            {
-                "LockIndex": 0,
-                "Key": "plugins/authz",
-                "Flags": 0,
-                "Value": "eyJjb250YWluZXIiOiAid3Vrb25nc3VuL21vb25fYXV0aHo6djQuMyIsICJwb3J0IjogODA4MX0=",
-                "CreateIndex": 14,
-                "ModifyIndex": 656
-            }
-        ],
-    )
+    # m.register_uri(
+    #     'GET', 'http://consul:8500/v1/kv/plugins?recurse=true',
+    #     json=[
+    #         {
+    #             "LockIndex": 0,
+    #             "Key": "plugins/authz",
+    #             "Flags": 0,
+    #             "Value": "eyJjb250YWluZXIiOiAid3Vrb25nc3VuL21vb25fYXV0aHo6djQuMyIsICJwb3J0IjogODA4MX0=",
+    #             "CreateIndex": 14,
+    #             "ModifyIndex": 656
+    #         }
+    #     ],
+    # )
 
 
 def register_pdp(m):
diff --git a/moon_orchestrator/tests/unit_python/utilities.py b/moon_orchestrator/tests/unit_python/utilities.py
index d64e4c7b..bc4aebcc 100644
--- a/moon_orchestrator/tests/unit_python/utilities.py
+++ b/moon_orchestrator/tests/unit_python/utilities.py
@@ -37,21 +37,19 @@ CONF = {
             "container": "wukongsun/moon_orchestrator:v4.3",
             "hostname": "interface"
         },
-        "interface": {
-            "bind": "0.0.0.0",
-            "port": 8080,
-            "container": "wukongsun/moon_interface:v4.3",
-            "hostname": "interface"
-        }
-    },
-    "plugins": {
-        "session": {
-            "port": 8082,
-            "container": "asteroide/session:latest"
-        },
-        "authz": {
-            "port": 8081,
-            "container": "wukongsun/moon_authz:v4.3"
+        "pipeline": {
+            "interface": {
+                "bind": "0.0.0.0",
+                "port": 8080,
+                "container": "wukongsun/moon_interface:v4.3",
+                "hostname": "interface"
+            },
+            "authz": {
+                "bind": "0.0.0.0",
+                "port": 8081,
+                "container": "wukongsun/moon_authz:v4.3",
+                "hostname": "authz"
+            },
         }
     },
     "logging": {