diff options
Diffstat (limited to 'moonv4/moon_orchestrator/moon_orchestrator')
16 files changed, 1049 insertions, 0 deletions
diff --git a/moonv4/moon_orchestrator/moon_orchestrator/__init__.py b/moonv4/moon_orchestrator/moon_orchestrator/__init__.py new file mode 100644 index 00000000..903c6518 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/__init__.py @@ -0,0 +1,6 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +__version__ = "0.1.0" diff --git a/moonv4/moon_orchestrator/moon_orchestrator/__main__.py b/moonv4/moon_orchestrator/moon_orchestrator/__main__.py new file mode 100644 index 00000000..b1feff49 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/__main__.py @@ -0,0 +1,3 @@ +from moon_orchestrator.server import main + +main() diff --git a/moonv4/moon_orchestrator/moon_orchestrator/api/__init__.py b/moonv4/moon_orchestrator/moon_orchestrator/api/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/api/__init__.py diff --git a/moonv4/moon_orchestrator/moon_orchestrator/api/configuration.py b/moonv4/moon_orchestrator/moon_orchestrator/api/configuration.py new file mode 100644 index 00000000..36c1f60d --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/api/configuration.py @@ -0,0 +1,63 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import json +from oslo_config import cfg +from oslo_log import log as logging +from moon_db.core import IntraExtensionRootManager +from moon_db.core import ConfigurationManager + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class Configuration(object): + """ + Retrieve the global configuration. + """ + + __version__ = "0.1.0" + + def get_policy_templates(self, ctx, args): + """List all policy templates + + :param ctx: {"id": "intra_extension_id"} + :param args: {} + :return: { + "template_id": { + "name": "name of the template", + "description": "description of the template", + } + """ + templates = ConfigurationManager.get_policy_templates_dict(ctx["user_id"]) + return {"policy_templates": templates} + + def get_aggregation_algorithms(self, ctx, args): + """List all aggregation algorithms + + :param ctx: {"id": "intra_extension_id"} + :param args: {} + :return: { + "algorithm_id": { + "name": "name of the algorithm", + "description": "description of the algorithm", + } + } + """ + return {'aggregation_algorithms': ConfigurationManager.get_aggregation_algorithms_dict(ctx["user_id"])} + + def get_sub_meta_rule_algorithms(self, ctx, args): + """List all sub meta rule algorithms + + :param ctx: {"id": "intra_extension_id"} + :param args: {} + :return: { + "algorithm_id": { + "name": "name of the algorithm", + "description": "description of the algorithm", + } + } + """ + return {'sub_meta_rule_algorithms': ConfigurationManager.get_sub_meta_rule_algorithms_dict(ctx["user_id"])} diff --git a/moonv4/moon_orchestrator/moon_orchestrator/api/containers.py b/moonv4/moon_orchestrator/moon_orchestrator/api/containers.py new file mode 100644 index 00000000..3572d615 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/api/containers.py @@ -0,0 +1,152 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import hashlib +from oslo_config import cfg +from oslo_log import log as logging +# from moon_db.core import IntraExtensionRootManager +# from moon_db.core import ConfigurationManager +from moon_utilities.security_functions import call + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class Containers(object): + """ + Manage containers. + """ + + __version__ = "0.1.0" + + def __init__(self, docker_manager): + self.docker_manager = docker_manager + self.components = dict() + for pdp_key, pdp_value in call("moon_manager", method="get_pdp", + ctx={"user_id": "admin", "id": None})["pdps"].items(): + self.add_container(ctx={"id": pdp_key, "pipeline": pdp_value["security_pipeline"]}) + + # for _ext_id, _ext_value in self.__get_pdp({"user_id": "admin"}, None)["intra_extensions"].items(): + # self.docker_manager.load(component="policy", uuid=_ext_id) + # # FIXME (asteroide): there may be other security_function here (delegation, ...) + # LOG.info("ADDING Containers {}".format(_ext_value)) + # self.docker_manager.load(component="function", uuid="{}_{}_{}".format("authz", "rbac_rule", _ext_id)) + + # def __get_pdp(self, ctx, args=None): + # """Get information about all pdp + # + # :param ctx: { + # "user_id": "uuid of a user", + # "id": "uuid of a tenant or an intra_extension" + # } + # :param args: {} + # :return: { + # "intra_extension_id": { + # "name": "name of the intra extension", + # "model": "model of the intra extension", + # "genre": "genre of the intra extension", + # "description": "description of the intra-extension" + # } + # } + # """ + # # TODO (asteroide): check if ctx["id"] is a tenant UUID or an intra_extension UUID. + # _ext = IntraExtensionRootManager.get_intra_extensions_dict(ctx["user_id"]) + # if ctx and "id" in ctx and ctx["id"]: + # if ctx["id"] in _ext: + # return {"pdp": {ctx["id"]: _ext[ctx["id"]]}} + # return {"error": "No pdp with id {}".format(ctx["id"])} + # return {"pdp": _ext} + + def get_container(self, ctx, args=None): + uuid = ctx.get("id") + keystone_project_id = ctx.get("keystone_project_id") + # _containers = self.docker_manager.get_component(uuid=uuid) + # LOG.info("containers={}".format(_containers)) + if uuid: + return self.components[uuid] + elif keystone_project_id: + for container_id, container_value in self.components.items(): + if container_value['keystone_project_id'] == keystone_project_id: + return {container_id: container_value} + else: + return {} + return {"containers": self.components} + + def add_container(self, ctx, args=None): + """Add containers linked to an intra-extension + + :param ctx: {"id": "intra_extension_uuid"} + :param args: {} + :return: { + "container_id1": {"status": True}, + "container_id2": {"status": True}, + } + """ + LOG.info("add_container {}".format(ctx)) + pdp = call("moon_manager", method="get_pdp", + ctx={"user_id": "admin", "id": ctx["id"]}, + args={})["pdps"] + pdp_id = list(pdp.keys())[0] + if not pdp[pdp_id]["keystone_project_id"]: + return {"result": "False", "message": "Cannot find keystone_project_id in pdp"} + keystone_project_id = pdp[pdp_id]["keystone_project_id"] + self.components[ctx["id"]] = [] + for policy_key, policy_value in call("moon_manager", method="get_policies", + ctx={"user_id": "admin", "id": None}, + args={})["policies"].items(): + if policy_key in ctx["pipeline"]: + models = call("moon_manager", method="get_models", + ctx={"user_id": "admin", "id": None}, + args={})["models"] + for meta_rule in models[policy_value['model_id']]['meta_rules']: + genre = policy_value['genre'] + pre_container_id = "pdp:{}_metarule:{}_project:{}".format(ctx["id"], meta_rule, keystone_project_id) + policy_component = self.docker_manager.load(component=genre, + uuid=pre_container_id) + self.components[ctx["id"]].append({ + "meta_rule_id": meta_rule, + "genre": policy_value['genre'], + "keystone_project_id": keystone_project_id, + "container_id": "authz_"+hashlib.sha224(pre_container_id.encode("utf-8")).hexdigest() + }) + return {"containers": self.components[ctx["id"]]} + # function_components = [] + # for pdp in ctx['pdp_pipeline']: + # key, value = pdp.split(":") + # LOG.info("add_container {}:{}".format(key, value)) + # function_components.append(self.docker_manager.load(component="function", + # uuid="{}_{}_{}".format(key, value, ctx["id"]))) + # containers = dict() + # containers[policy_component.id] = policy_component.get_status() + # for component in function_components: + # containers[component.id] = component.get_status() + # return {"containers": containers} + + def delete_container(self, ctx, args=None): + """Delete a container + + :param ctx: {"id": "intra_extension_uuid"} + :param args: {} + :return: {} + """ + try: + self.docker_manager.kill(component_id="moon_secpolicy_"+ctx["id"]) + try: + # FIXME (asteroide): need to select other security_function here + self.docker_manager.kill(component_id="moon_secfunction_authz_"+ctx["id"]) + except Exception as e: + LOG.error(e, exc_info=True) + return {"result": True, + "error": {'code': 200, 'title': 'Moon Warning', 'description': str(e)}, + "intra_extension_id": ctx["id"], + "ctx": ctx, "args": args} + except Exception as e: + LOG.error(e, exc_info=True) + return {"result": False, + "error": {'code': 500, 'title': 'Moon Error', 'description': str(e)}, + "intra_extension_id": ctx["id"], + "ctx": ctx, "args": args} + return {"result": True} + diff --git a/moonv4/moon_orchestrator/moon_orchestrator/api/generic.py b/moonv4/moon_orchestrator/moon_orchestrator/api/generic.py new file mode 100644 index 00000000..cadd98d3 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/api/generic.py @@ -0,0 +1,29 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + + +class Status(object): + """ + Retrieve the current status of all components. + """ + + __version__ = "0.1.0" + + def get_status(self, ctx, args): + """Retrieve the current status of all components.""" + return {"status": "Running"} + + +class Logs(object): + """ + Retrieve the current status of all components. + """ + + __version__ = "0.1.0" + + def get_logs(self, ctx, args): + return {"error": "NotImplemented", "ctx": ctx, "args": args} + + diff --git a/moonv4/moon_orchestrator/moon_orchestrator/api/slaves.py b/moonv4/moon_orchestrator/moon_orchestrator/api/slaves.py new file mode 100644 index 00000000..66ddf256 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/api/slaves.py @@ -0,0 +1,76 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +from oslo_config import cfg +from oslo_log import log as logging +from uuid import uuid4 + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class Slaves(object): + """ + Manage containers. + """ + + __version__ = "0.1.0" + + def __init__(self, slaves): + self.slaves = slaves + + def add_slave(self, ctx, args=None): + """Add a new slave in the global list + + :param ctx: { + "name": "name of the slave", + "description": "description" + } + :param args: {} + :return: { + "uuid_of_the_slave": { + "name": "name of the slave", + "description": "description" + } + } + """ + if "name" in ctx: + for _id, _dict in self.slaves.items(): + if _dict['name'] == ctx['name']: + LOG.warning("A slave named {} already exists!".format(ctx['name'])) + return {"slaves": {_id: _dict}} + uuid = uuid4().hex + ctx.pop("method") + ctx.pop("call_master") + self.slaves[uuid] = ctx + return {"slaves": {uuid: ctx}} + + def get_slaves(self, ctx, args=None): + """Get all the known slaves + + :param ctx: {} + :param args: {} + :return: { + "uuid_of_the_slave": { + "name": "name of the slave", + "description": "description" + } + } + """ + return {"slaves": self.slaves} + + def delete_slave(self, ctx, args=None): + """Delete a previous slave in the global list + + :param ctx: { + "id": "ID of the slave" + } + :param args: {} + :return: None + """ + if "id" in ctx: + if ctx['id'] in self.slaves: + self.slaves.pop(ctx['id']) + return {"slaves": self.slaves} diff --git a/moonv4/moon_orchestrator/moon_orchestrator/dockers.py b/moonv4/moon_orchestrator/moon_orchestrator/dockers.py new file mode 100644 index 00000000..2eecdc0e --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/dockers.py @@ -0,0 +1,191 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import os +import json +import glob +import uuid +import shutil +import errno +from uuid import uuid4 +from oslo_config import cfg +from oslo_log import log as logging +from jinja2 import FileSystemLoader, Environment +from moon_utilities.options import get_docker_template_dir + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +DOMAIN = "moon_orchestrator" + +__CWD__ = os.path.dirname(os.path.abspath(__file__)) +TEMPLATES_FOLDER = get_docker_template_dir() + + +class DockerBase: + + docker = None + image_id = None + tag = 'moon/component' + tmp_dir = os.path.join("/tmp", uuid.uuid4().hex) + name = "" + __build = """RUN mkdir -p /etc/moon/ +COPY conf /etc/moon/ +ADD dist/{py_pkg}.tar.gz /root +WORKDIR /root/{py_pkg} +RUN pip3 install --upgrade -r requirements.txt +RUN pip3 install --upgrade . +""" + + def __init__(self, + name, + run_cmd, + host=None, + build_cmd=None, + conf_file="", + id=None, + docker=None, + network_config=None, + tag="", + port=None + ): + self.conf_file = conf_file + self.docker = docker + self.network_config = network_config + self.name = name + self.id = id if id else name + "_" + uuid4().hex + self.tag = "moon/{}".format(name) + self.build_cmd = build_cmd if build_cmd else self.__build + self.run_cmd = run_cmd + self.host = host + self.docker_id = id + self.port = port + containers = self.docker.containers() + if self.id not in map(lambda x: x['Id'], containers): + self.create_container(tag) + self.run_docker() + else: + LOG.info("Component {} already running...".format(name)) + + def create_container(self, container=None): + if not container: + proxy = CONF.proxy + if CONF.proxy: + proxy = "ENV http_proxy {0}\nENV https_proxy {0}\n".format(CONF.proxy) + run = self.build_cmd.format( + py_pkg=self.__get_last_version_of_pkg(self.name).replace(".tar.gz", "").replace("dist/", ""), + port=self.port + ) + docker_str = self.__get_template().render(run=run, cmd=self.run_cmd, proxy=proxy) + self.__create_tmp_dir(docker_str) + self.create_docker(docker_str) + else: + self.tag = container + + def __create_tmp_dir(self, docker_str): + try: + os.mkdir(self.tmp_dir) + except OSError as e: + LOG.warning("Problem when creating temporary directory ({})".format(e)) + + try: + os.mkdir(os.path.join(self.tmp_dir, "dist")) + except OSError as e: + LOG.warning("Problem when creating temporary directory ({})".format(e)) + for _file in glob.glob("{}/*".format(CONF.dist_dir)): + LOG.info("Copying {}".format(_file)) + shutil.copy(_file, os.path.join(self.tmp_dir, "dist")) + + try: + shutil.copytree(os.path.dirname(self.conf_file), os.path.join(self.tmp_dir, "conf")) + except OSError as exc: + if exc.errno == errno.ENOTDIR: + shutil.copy(os.path.dirname(self.conf_file), os.path.join(self.tmp_dir, "conf")) + elif exc.errno == errno.EEXIST: + pass + else: + LOG.info("exc.errno = {}".format(exc.errno)) + raise + + open("{}/Dockerfile".format(self.tmp_dir), "w").write(docker_str) + + def __get_docker_network(self, name="moon"): + if self.host: + return self.docker.create_networking_config({ + name: self.docker.create_endpoint_config( + aliases=[self.id, ], + ipv4_address=self.host, + ) + }) + else: + return self.docker.create_networking_config({ + name: self.docker.create_endpoint_config( + aliases=[self.id, ] + ) + }) + + @staticmethod + def __get_last_version_of_pkg(name): + files = [] + for filename in glob.glob("{}/{}*".format(CONF.dist_dir, name)): + files.append(filename) + files.sort() + try: + return os.path.basename(files[-1]) + except Exception as e: + LOG.error("__get_last_version_of_pkg {}/{}*".format(CONF.dist_dir, name)) + raise e + + def run_docker(self): + LOG.info("run_docker hostname={}".format(self.id.replace("_", "-"))) + if self.port: + host_config = self.docker.create_host_config(port_bindings={ + self.port: self.port + }) + else: + host_config = self.docker.create_host_config() + + output = self.docker.create_container(image=self.tag, + command=list(self.run_cmd), + hostname=str(self.id.replace("_", "-")), + name=str(self.id), + networking_config=self.__get_docker_network(), + host_config=host_config + ) + container_data = self.docker.inspect_container(output['Id']) + name = container_data["Name"] + LOG.info("Running container {} with ID {}".format(self.tag, output)) + LOG.info("output id = {}".format(output['Id'])) + self.docker.start(container=output['Id']) + LOG.info("Running container output {}".format(self.docker.logs( + container=name, + # stdout=True, + # stderr=True + ).decode("utf-8"))) + self.name = name + self.docker_id = output['Id'] + + def create_docker(self, docker_str): + # f = BytesIO(docker_str.encode('utf-8')) + LOG.info("Building {}".format(self.tmp_dir)) + # TODO (dthom): halt on built errors (or emit a log) + _output = self.docker.build(path=self.tmp_dir, rm=True, tag=self.tag) + # _output = self.cli.build(fileobj=f, rm=True, tag=self.tag, stream=True) + for line in _output: + jline = json.loads(line.decode("utf-8")) + if "stream" in jline: + LOG.info("\033[33m" + jline["stream"].strip() + "\033[m") + else: + LOG.info("\033[33m" + str(jline).strip() + "\033[m") + else: + LOG.debug(_output) + LOG.info("tag = {}".format(self.tag)) + LOG.info("images = {}".format(self.docker.images(name=self.tag))) + self.image_id = self.docker.images(name=self.tag)[0]['Id'] + + @staticmethod + def __get_template(filename="template.dockerfile"): + simple_loader = FileSystemLoader(TEMPLATES_FOLDER) + env = Environment(loader=simple_loader) + return env.get_template(filename) diff --git a/moonv4/moon_orchestrator/moon_orchestrator/messenger.py b/moonv4/moon_orchestrator/moon_orchestrator/messenger.py new file mode 100644 index 00000000..6b54255f --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/messenger.py @@ -0,0 +1,84 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +from oslo_config import cfg +import oslo_messaging +from oslo_log import log as logging +import time +from moon_utilities.api import APIList +from moon_utilities.security_functions import call +from moon_utilities.exceptions import RootPDPNotInitialized + +from oslo_config import cfg +from moon_orchestrator.api.generic import Status, Logs +# from moon_orchestrator.api.configuration import Configuration +from moon_orchestrator.api.containers import Containers +from moon_orchestrator.api.slaves import Slaves + +TOPIC = "orchestrator" +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class Server: + + def __init__(self, containers, docker_manager, slaves): + self.CONTAINERS = containers + self.transport = oslo_messaging.get_transport(cfg.CONF) + self.target = oslo_messaging.Target(topic=TOPIC, server='server1') + LOG.info("Starting MQ server with topic: {}".format(TOPIC)) + self.docker_manager = docker_manager + for _container in containers: + Status._container = containers[_container] + self.endpoints = [ + APIList((Status, Logs, Containers)), + Status(), + Logs(), + Containers(self.docker_manager), + # Configuration(), + Slaves(slaves) + ] + self.server = oslo_messaging.get_rpc_server(self.transport, self.target, self.endpoints, + executor='threading', + access_policy=oslo_messaging.DefaultRPCAccessPolicy) + + # @staticmethod + # def __check_root_pdp(): + # root_exist = False + # for key, value in call("manager", ctx={"user_id": "admin"}, + # method="get_pdp", args={})["pdp"].items(): + # if value["name"] == CONF.root_policy_directory: + # root_exist = True + # if not root_exist: + # ie = call("manager", ctx={"user_id": "admin"}, method="add_pdp", + # args={ + # "name": "policy_root", + # "model": CONF.root_policy_directory, + # "genre": "admin", + # "description": "policy_root" + # }) + # if "result" in ie and not ie["result"]: + # raise RootPDPNotInitialized + + def run(self): + try: + self.server.start() + # try: + # raise NotImplementedError + # # self.__check_root_pdp() + # except Exception as e: + # LOG.error("Exception occurred when creating Root PDP: {}".format(e)) + while True: + time.sleep(1) + except KeyboardInterrupt: + LOG.warning("Stopping server by crtl+c (please be patient, closing connections...)") + except SystemExit: + LOG.warning("Stopping server (please be patient, closing connections...)") + except Exception as e: + LOG.error("Exception occurred: {}".format(e)) + + self.server.stop() + self.server.wait() + diff --git a/moonv4/moon_orchestrator/moon_orchestrator/scoper.py b/moonv4/moon_orchestrator/moon_orchestrator/scoper.py new file mode 100644 index 00000000..ebfb12f8 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/scoper.py @@ -0,0 +1,40 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging +from moon_orchestrator.dockers import DockerBase + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +DOMAIN = "moon_orchestrator" + + +class Scoper(DockerBase): + + id = "moon_scoper" + + def __init__(self, conf_file="", docker=None, network_config=None): + super(Scoper, self).__init__( + name="moon_scoper", + id=self.id, + run_cmd=["python3", "-m", "moon_scoper"], + host=CONF.scoper.host, + conf_file=conf_file, + docker=docker, + network_config=network_config, + tag=CONF.scoper.container + ) + + @staticmethod + def get_status(): + transport = oslo_messaging.get_transport(CONF) + target = oslo_messaging.Target(topic='scoper', version='1.0') + client = oslo_messaging.RPCClient(transport, target) + LOG.info("Calling Status on scoper component...") + ret = client.call({"component_id": "scoper"}, 'get_status', args=None) + LOG.info(ret) + return ret diff --git a/moonv4/moon_orchestrator/moon_orchestrator/security_function.py b/moonv4/moon_orchestrator/moon_orchestrator/security_function.py new file mode 100644 index 00000000..1b33ef60 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/security_function.py @@ -0,0 +1,56 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import os +import time +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging +from moon_orchestrator.dockers import DockerBase + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +DOMAIN = "moon_orchestrator" + +__CWD__ = os.path.dirname(os.path.abspath(__file__)) +# TODO (dthom): select the right template folder +TEMPLATES_FOLDER = os.path.join(__CWD__, "..", "conf", "dockers") + + +class SecurityFunction(DockerBase): + + id = "moon_function" + __build = """RUN mkdir -p /etc/moon/ +COPY conf /etc/moon/ +ADD dist/{py_pkg}.tar.gz /root +WORKDIR /root/{py_pkg} +RUN pip3 install -r requirements.txt +RUN pip3 install . +""" + + def __init__(self, intra_extension_id, conf_file="", docker=None, network_config=None): + self.id = "moon_pdp_{}".format(intra_extension_id) + super(SecurityFunction, self).__init__( + name="moon_secfunction", + run_cmd=["python3", "-m", "moon_secfunction", intra_extension_id], + conf_file=conf_file, + docker=docker, + network_config=network_config, + build_cmd=self.__build, + id=self.id, + tag=CONF.security_function.container + ) + # note(asteroide): time to let the new docker boot + time.sleep(3) + self.get_status() + + def get_status(self): + transport = oslo_messaging.get_transport(CONF) + target = oslo_messaging.Target(topic=self.id, version='1.0') + client = oslo_messaging.RPCClient(transport, target) + LOG.info("Calling Status on {}".format(self.id)) + ret = client.call({"component_id": self.id}, 'get_status', args=None) + LOG.info(ret) + return ret diff --git a/moonv4/moon_orchestrator/moon_orchestrator/security_interface.py b/moonv4/moon_orchestrator/moon_orchestrator/security_interface.py new file mode 100644 index 00000000..656c4340 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/security_interface.py @@ -0,0 +1,45 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import os +from oslo_config import cfg +from oslo_log import log as logging +from moon_orchestrator.dockers import DockerBase + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +DOMAIN = "moon_orchestrator" + +__CWD__ = os.path.dirname(os.path.abspath(__file__)) +# TODO (dthom): select the right template folder +TEMPLATES_FOLDER = os.path.join(__CWD__, "..", "conf", "dockers") + + +class SecurityInterface(DockerBase): + + id = "moon_interface" + __build = """RUN mkdir -p /etc/moon/ + COPY conf /etc/moon/ + ADD dist/{py_pkg}.tar.gz /root + WORKDIR /root/{py_pkg} + RUN pip3 install -r requirements.txt + RUN pip3 install . + EXPOSE {port} + """ + + def __init__(self, conf_file="", docker=None, network_config=None): + super(SecurityInterface, self).__init__( + name="moon_interface", + id=self.id, + run_cmd=["python3", "-m", "moon_interface"], + host=CONF.interface.host, + conf_file=conf_file, + docker=docker, + network_config=network_config, + tag=CONF.interface.container, + build_cmd=self.__build, + port=CONF.interface.port + ) + diff --git a/moonv4/moon_orchestrator/moon_orchestrator/security_manager.py b/moonv4/moon_orchestrator/moon_orchestrator/security_manager.py new file mode 100644 index 00000000..c7dc4c63 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/security_manager.py @@ -0,0 +1,56 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import os +import time +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging +from moon_orchestrator.dockers import DockerBase + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +DOMAIN = "moon_orchestrator" + +__CWD__ = os.path.dirname(os.path.abspath(__file__)) +# TODO (dthom): select the right template folder +TEMPLATES_FOLDER = os.path.join(__CWD__, "..", "conf", "dockers") + + +class SecurityManager(DockerBase): + + id = "moon_manager" + __build = """RUN mkdir -p /etc/moon/ +COPY conf /etc/moon/ +ADD dist/{py_pkg}.tar.gz /root +WORKDIR /root/{py_pkg} +RUN pip3 install -r requirements.txt +RUN pip3 install . +""" + + def __init__(self, conf_file="", docker=None, network_config=None): + self.id = "moon_manager" + super(SecurityManager, self).__init__( + name="moon_manager", + run_cmd=["python3", "-m", "moon_manager"], + conf_file=conf_file, + docker=docker, + network_config=network_config, + build_cmd=self.__build, + id=self.id, + tag=CONF.security_manager.container + ) + # note(asteroide): time to let the new docker boot + time.sleep(3) + self.get_status() + + def get_status(self): + transport = oslo_messaging.get_transport(CONF) + target = oslo_messaging.Target(topic=self.id, version='1.0') + client = oslo_messaging.RPCClient(transport, target) + LOG.info("Calling Status on {}".format(self.id)) + ret = client.call({"component_id": self.id}, 'get_status', args=None) + LOG.info(ret) + return ret diff --git a/moonv4/moon_orchestrator/moon_orchestrator/security_policy.py b/moonv4/moon_orchestrator/moon_orchestrator/security_policy.py new file mode 100644 index 00000000..5cb1d51c --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/security_policy.py @@ -0,0 +1,56 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import os +import time +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging +from moon_orchestrator.dockers import DockerBase + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +DOMAIN = "moon_orchestrator" + +__CWD__ = os.path.dirname(os.path.abspath(__file__)) +# TODO (dthom): select the right template folder +TEMPLATES_FOLDER = os.path.join(__CWD__, "..", "conf", "dockers") + + +class SecurityPolicy(DockerBase): + + id = "moon_secpolicy" + __build = """RUN mkdir -p /etc/moon/ +COPY conf /etc/moon/ +ADD dist/{py_pkg}.tar.gz /root +WORKDIR /root/{py_pkg} +RUN pip3 install -r requirements.txt +RUN pip3 install . +""" + + def __init__(self, intra_extension_id, conf_file="", docker=None, network_config=None): + self.id = "moon_secpolicy" + intra_extension_id + super(SecurityPolicy, self).__init__( + name="moon_secpolicy", + run_cmd=["python3", "-m", "moon_secpolicy", intra_extension_id], + conf_file=conf_file, + docker=docker, + network_config=network_config, + build_cmd=self.__build, + id="moon_secpolicy_{}".format(intra_extension_id), + tag=CONF.security_policy.container + ) + # note(asteroide): time to let the new docker boot + time.sleep(3) + self.get_status() + + def get_status(self): + transport = oslo_messaging.get_transport(CONF) + target = oslo_messaging.Target(topic=self.id, version='1.0') + client = oslo_messaging.RPCClient(transport, target) + LOG.info("Calling Status on {}".format(self.id)) + ret = client.call({"component_id": self.id}, 'get_status', args=None) + LOG.info(ret) + return ret diff --git a/moonv4/moon_orchestrator/moon_orchestrator/security_router.py b/moonv4/moon_orchestrator/moon_orchestrator/security_router.py new file mode 100644 index 00000000..290ab111 --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/security_router.py @@ -0,0 +1,52 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import os +import json +import glob +import uuid +import shutil +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging +from io import BytesIO +from docker import Client +from jinja2 import FileSystemLoader, Environment +from moon_orchestrator.dockers import DockerBase + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +DOMAIN = "moon_orchestrator" + +__CWD__ = os.path.dirname(os.path.abspath(__file__)) +# TODO (dthom): select the right template folder +TEMPLATES_FOLDER = os.path.join(__CWD__, "..", "conf", "dockers") + + +class SecurityRouter(DockerBase): + + id = "moon_router" + + def __init__(self, conf_file="", docker=None, network_config=None): + super(SecurityRouter, self).__init__( + name="moon_secrouter", + id=self.id, + run_cmd=["python3", "-m", "moon_secrouter"], + host=CONF.security_router.host, + conf_file=conf_file, + docker=docker, + network_config=network_config, + tag=CONF.security_router.container + ) + + @staticmethod + def get_status(): + transport = oslo_messaging.get_transport(CONF) + target = oslo_messaging.Target(topic='security_router', version='1.0') + client = oslo_messaging.RPCClient(transport, target) + LOG.info("Calling Status on security_server...") + ret = client.call({"component_id": "security_router"}, 'get_status', args=None) + LOG.info(ret) + return ret diff --git a/moonv4/moon_orchestrator/moon_orchestrator/server.py b/moonv4/moon_orchestrator/moon_orchestrator/server.py new file mode 100644 index 00000000..4fc9d5fd --- /dev/null +++ b/moonv4/moon_orchestrator/moon_orchestrator/server.py @@ -0,0 +1,140 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import sys +import os +import hashlib +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging +from docker import Client +import docker.errors as docker_errors +from importlib.machinery import SourceFileLoader +from moon_utilities import options +from moon_orchestrator.security_router import SecurityRouter +from moon_orchestrator.security_interface import SecurityInterface +from moon_orchestrator.security_manager import SecurityManager +from moon_orchestrator.security_function import SecurityFunction +# from moon_orchestrator.security_policy import SecurityPolicy +# from moon_orchestrator.security_function import SecurityFunction +from moon_orchestrator import messenger + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + +CONTAINERS = {} +SLAVES = {} +docker = Client(base_url=CONF.docker_url) + + +# def get_template(filename="template.dockerfile"): +# simple_loader = FileSystemLoader(TEMPLATES_FOLDER) +# env = Environment(loader=simple_loader) +# return env.get_template(filename) + + +def create_docker_network(name="moon"): + + return docker.create_networking_config({ + name: docker.create_endpoint_config(), + 'aliases': ['orchestrator', ] + }) + + +def load_plugin(plugname): + try: + m = SourceFileLoader("scenario", os.path.join(CONF.plugin_dir, plugname+".py")) + return m.load_module() + except ImportError as e: + LOG.error("Error in importing plugin {}".format(plugname)) + LOG.error("{}".format(e)) + + +class DockerManager: + + @staticmethod + def load(component, uuid): + """Load a new docker mapping the component given + + :param component: the name of the component (policy or function) + :param uuid: the uuid of the intra_extension linked to that component + :return: the created component + """ + component_id = "authz_"+hashlib.sha224(uuid.encode("utf-8")).hexdigest() + if component_id not in CONTAINERS: + plug = load_plugin(component) + LOG.info("Creating {} with id {}".format(component, uuid)) + component = plug.run(uuid, options.filename, docker=docker, network_config=create_docker_network()) + CONTAINERS[component_id] = component + return component + + @staticmethod + def get_component(uuid=None): + if uuid: + return CONTAINERS.get(uuid, None) + return CONTAINERS + + @staticmethod + def kill(component_id, delete=True): + LOG.info("Killing container {}".format(component_id)) + docker.kill(container=component_id) + if delete: + docker.remove_container(container=component_id) + + +def _exit(exit_number=0, docker=None, error=None): + for _container in CONTAINERS: + LOG.warning("Deleting containers named {}...".format(_container)) + # print(40 * "-" + _container) + try: + # print(docker.logs(container=_container).decode("utf-8")) + docker.kill(container=_container) + except docker_errors.NotFound: + LOG.error("The container {} was not found".format(_container)) + except docker_errors.APIError as e: + LOG.error(e) + else: + docker.remove_container(container=_container) + + # TODO (dthom): put in the debug log + if error: + LOG.info(str(error)) + sys.exit(exit_number) + + +def main(): + # conf_file = options.configure(DOMAIN) + LOG.info("Starting server with IP {}".format(CONF.orchestrator.host)) + + docker_manager = DockerManager() + + network_config = create_docker_network() + + LOG.info("Creating Security Router") + router = SecurityRouter(options.filename, docker=docker, network_config=network_config) + CONTAINERS[router.id] = router + + LOG.info("Creating Manager") + manager = SecurityManager(options.filename, docker=docker, network_config=network_config) + CONTAINERS[manager.id] = manager + + LOG.info("Creating Security Interface") + interface = SecurityInterface(options.filename, docker=docker, network_config=network_config) + CONTAINERS[interface.id] = interface + + try: + router.get_status() + except oslo_messaging.rpc.client.RemoteError as e: + LOG.error("Cannot check status of remote container!") + _exit(1, docker, e) + serv = messenger.Server(containers=CONTAINERS, docker_manager=docker_manager, slaves=SLAVES) + try: + serv.run() + finally: + _exit(0, docker) + + +if __name__ == '__main__': + main() |