From 0e678794fd2c858069fbbe0b985a0e2f064862b1 Mon Sep 17 00:00:00 2001 From: Thomas Duval Date: Wed, 3 Jan 2018 15:17:26 +0100 Subject: Clean the code Change-Id: I6693e1e2c648791027d290f64798482d655ad011 --- moon_orchestrator/moon_orchestrator/__init__.py | 2 +- moon_orchestrator/moon_orchestrator/__main__.py | 4 +- moon_orchestrator/moon_orchestrator/api/generic.py | 41 +----------- moon_orchestrator/moon_orchestrator/api/pods.py | 11 ++-- moon_orchestrator/moon_orchestrator/drivers.py | 22 +++---- moon_orchestrator/moon_orchestrator/http_server.py | 75 +++++++++++----------- moon_orchestrator/moon_orchestrator/server.py | 22 ++++--- moon_orchestrator/requirements.txt | 1 - moon_orchestrator/tests/unit_python/test_pods.py | 4 +- 9 files changed, 77 insertions(+), 105 deletions(-) (limited to 'moon_orchestrator') diff --git a/moon_orchestrator/moon_orchestrator/__init__.py b/moon_orchestrator/moon_orchestrator/__init__.py index 2302dea9..3276f42d 100644 --- a/moon_orchestrator/moon_orchestrator/__init__.py +++ b/moon_orchestrator/moon_orchestrator/__init__.py @@ -3,4 +3,4 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -__version__ = "1.1.0" +__version__ = "4.3.1" diff --git a/moon_orchestrator/moon_orchestrator/__main__.py b/moon_orchestrator/moon_orchestrator/__main__.py index 9ebc3a7f..df051f26 100644 --- a/moon_orchestrator/moon_orchestrator/__main__.py +++ b/moon_orchestrator/moon_orchestrator/__main__.py @@ -1,4 +1,4 @@ -from moon_orchestrator.server import main +from moon_orchestrator.server import create_server -server = main() +server = create_server() server.run() diff --git a/moon_orchestrator/moon_orchestrator/api/generic.py b/moon_orchestrator/moon_orchestrator/api/generic.py index 84de4e69..37e30a22 100644 --- a/moon_orchestrator/moon_orchestrator/api/generic.py +++ b/moon_orchestrator/moon_orchestrator/api/generic.py @@ -11,9 +11,9 @@ import logging import moon_orchestrator.api from python_moonutilities.security_functions import check_auth -__version__ = "0.1.0" +__version__ = "4.3.1" -LOG = logging.getLogger("moon.orchestrator.api." + __name__) +logger = logging.getLogger("moon.orchestrator.api." + __name__) class Status(Resource): @@ -38,41 +38,6 @@ class Status(Resource): raise NotImplemented -class Logs(Resource): - """ - Endpoint for logs requests - """ - - __urls__ = ("/logs", "/logs/", "/logs/") - - def get(self, component_id=None): - """Get logs from the Moon platform - - :param component_id: the ID of the component your are looking for (optional) - :return: [ - "2015-04-15-13:45:20 - "2015-04-15-13:45:21 - "2015-04-15-13:45:22 - "2015-04-15-13:45:23 - ] - """ - filter_str = request.args.get('filter', '') - from_str = request.args.get('from', '') - to_str = request.args.get('to', '') - event_number = request.args.get('event_number', '') - try: - event_number = int(event_number) - except ValueError: - event_number = None - args = dict() - args["filter"] = filter_str - args["from"] = from_str - args["to"] = to_str - args["event_number"] = event_number - - raise NotImplemented - - class API(Resource): """ Endpoint for API requests @@ -125,7 +90,7 @@ class API(Resource): if endpoint_id in api_desc[group_id]: return {group_id: {endpoint_id: api_desc[group_id][endpoint_id]}} elif len(endpoint_id) > 0: - LOG.error("Unknown endpoint_id {}".format(endpoint_id)) + logger.error("Unknown endpoint_id {}".format(endpoint_id)) return {"error": "Unknown endpoint_id {}".format(endpoint_id)} return {group_id: api_desc[group_id]} return api_desc diff --git a/moon_orchestrator/moon_orchestrator/api/pods.py b/moon_orchestrator/moon_orchestrator/api/pods.py index 9bca4d93..7b89372e 100644 --- a/moon_orchestrator/moon_orchestrator/api/pods.py +++ b/moon_orchestrator/moon_orchestrator/api/pods.py @@ -8,7 +8,7 @@ from flask_restful import Resource from python_moonutilities.security_functions import check_auth import logging -LOG = logging.getLogger("moon.orchestrator.api.pods") +logger = logging.getLogger("moon.orchestrator.api.pods") class Pods(Resource): @@ -16,6 +16,8 @@ class Pods(Resource): Endpoint for pdp requests """ + __version__ = "4.3.1" + __urls__ = ( "/pods", "/pods/", @@ -25,7 +27,7 @@ class Pods(Resource): def __init__(self, **kwargs): self.driver = kwargs.get("driver") - self.create_security_function = kwargs.get("create_security_function_hook") + self.create_pipeline = kwargs.get("create_pipeline_hook") @check_auth def get(self, uuid=None, user_id=None): @@ -43,7 +45,6 @@ class Pods(Resource): :internal_api: get_pdp """ pods = {} - # LOG.info("pods={}".format(self.driver.get_pods())) if uuid: return {"pods": self.driver.get_pods(uuid)} for _pod_key, _pod_values in self.driver.get_pods().items(): @@ -73,8 +74,8 @@ class Pods(Resource): } } """ - LOG.info("POST param={}".format(request.json)) - self.create_security_function( + logger.debug("POST param={}".format(request.json)) + self.create_pipeline( request.json.get("keystone_project_id"), request.json.get("pdp_id"), request.json.get("security_pipeline"), diff --git a/moon_orchestrator/moon_orchestrator/drivers.py b/moon_orchestrator/moon_orchestrator/drivers.py index 08c53be3..07f012aa 100644 --- a/moon_orchestrator/moon_orchestrator/drivers.py +++ b/moon_orchestrator/moon_orchestrator/drivers.py @@ -8,14 +8,14 @@ import logging import urllib3.exceptions from python_moonutilities import configuration -LOG = logging.getLogger("moon.orchestrator.drivers") +logger = logging.getLogger("moon.orchestrator.drivers") def get_driver(): try: return K8S() except urllib3.exceptions.MaxRetryError as e: - LOG.exception(e) + logger.exception(e) return Docker() @@ -60,12 +60,12 @@ class K8S(Driver): if name: pods = self.client.list_pod_for_all_namespaces(watch=False) for pod in pods.items: - LOG.info("get_pods {}".format(pod.metadata.name)) + logger.debug("get_pods {}".format(pod.metadata.name)) if name in pod.metadata.name: return pod else: return None - LOG.info("get_pods cache={}".format(self.cache)) + logger.debug("get_pods cache={}".format(self.cache)) return self.cache @staticmethod @@ -109,9 +109,7 @@ class K8S(Driver): ) resp = client.create_namespaced_deployment(body=pod_manifest, namespace='moon') - LOG.info("Pod {} created!".format(data[0].get('name'))) - # logger.info(yaml.dump(pod_manifest, sys.stdout)) - # logger.info(resp) + logger.info("Pod {} created!".format(data[0].get('name'))) return resp @staticmethod @@ -144,18 +142,18 @@ class K8S(Driver): service_manifest['spec']['type'] = "NodePort" resp = client.create_namespaced_service(namespace="moon", body=service_manifest) - LOG.info("Service {} created!".format(data.get('name'))) + logger.info("Service {} created!".format(data.get('name'))) return resp def load_pod(self, data, api_client=None, ext_client=None, expose=False): _client = api_client if api_client else self.client pod = self.__create_pod(client=ext_client, data=data) - service = self.__create_service(client=_client, data=data[0], + self.__create_service(client=_client, data=data[0], expose=expose) self.cache[pod.metadata.uid] = data def delete_pod(self, uuid=None, name=None): - LOG.info("Deleting pod {}".format(uuid)) + logger.info("Deleting pod {}".format(uuid)) # TODO: delete_namespaced_deployment # https://github.com/kubernetes-incubator/client-python/blob/master/kubernetes/client/apis/extensions_v1beta1_api.py @@ -167,9 +165,9 @@ class K8S(Driver): class Docker(Driver): def load_pod(self, data, api_client=None, ext_client=None): - LOG.info("Creating pod {}".format(data[0].get('name'))) + logger.info("Creating pod {}".format(data[0].get('name'))) raise NotImplementedError def delete_pod(self, uuid=None, name=None): - LOG.info("Deleting pod {}".format(uuid)) + logger.info("Deleting pod {}".format(uuid)) raise NotImplementedError diff --git a/moon_orchestrator/moon_orchestrator/http_server.py b/moon_orchestrator/moon_orchestrator/http_server.py index 62d785a2..7105ea7a 100644 --- a/moon_orchestrator/moon_orchestrator/http_server.py +++ b/moon_orchestrator/moon_orchestrator/http_server.py @@ -4,24 +4,22 @@ # or at 'http://www.apache.org/licenses/LICENSE-2.0'. from flask import Flask, jsonify -from flask_cors import CORS, cross_origin from flask_restful import Resource, Api -import logging from kubernetes import client, config -import random +import logging import requests import time from moon_orchestrator import __version__ from moon_orchestrator.api.pods import Pods -from moon_orchestrator.api.generic import Logs, Status +from moon_orchestrator.api.generic import Status +from moon_orchestrator.drivers import get_driver from python_moonutilities import configuration, exceptions from python_moonutilities.misc import get_random_name -from moon_orchestrator.drivers import get_driver -LOG = logging.getLogger("moon.orchestrator.http_server") +logger = logging.getLogger("moon.orchestrator.http_server") __API__ = ( - Status, Logs + Status, ) @@ -107,7 +105,7 @@ class HTTPServer(Server): # CORS(self.app) self.api = Api(self.app) self.driver = get_driver() - LOG.info("Driver = {}".format(self.driver.__class__)) + logger.info("Driver = {}".format(self.driver.__class__)) self.__set_route() self.__hook_errors() pdp = None @@ -117,20 +115,20 @@ class HTTPServer(Server): "http://{}:{}/pdp".format(self.manager_hostname, self.manager_port)) except requests.exceptions.ConnectionError: - LOG.warning("Manager is not ready, standby...") + logger.warning("Manager is not ready, standby...") time.sleep(1) except KeyError: - LOG.warning("Manager is not ready, standby...") + logger.warning("Manager is not ready, standby...") time.sleep(1) else: if "pdps" in pdp.json(): break - LOG.debug("pdp={}".format(pdp)) + logger.debug("pdp={}".format(pdp)) self.create_wrappers() for _pdp_key, _pdp_value in pdp.json()['pdps'].items(): if _pdp_value.get('keystone_project_id'): # TODO: select context to add security function - self.create_security_function( + self.create_pipeline( keystone_project_id=_pdp_value.get('keystone_project_id'), pdp_id=_pdp_key, policy_ids=_pdp_value.get('security_pipeline', [])) @@ -154,8 +152,8 @@ class HTTPServer(Server): self.api.add_resource(Pods, *Pods.__urls__, resource_class_kwargs={ "driver": self.driver, - "create_security_function_hook": - self.create_security_function, + "create_pipeline_hook": + self.create_pipeline, }) def run(self): @@ -167,8 +165,8 @@ class HTTPServer(Server): def create_wrappers(self): contexts, active_context = self.driver.get_slaves() - LOG.debug("contexts: {}".format(contexts)) - LOG.debug("active_context: {}".format(active_context)) + logger.debug("contexts: {}".format(contexts)) + logger.debug("active_context: {}".format(active_context)) conf = configuration.get_configuration("components/wrapper") hostname = conf["components/wrapper"].get( "hostname", "wrapper") @@ -178,7 +176,7 @@ class HTTPServer(Server): "wukongsun/moon_wrapper:v4.3") for _ctx in contexts: _config = config.new_client_from_config(context=_ctx['name']) - LOG.debug("_config={}".format(_config)) + logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) # TODO: get data from consul @@ -189,15 +187,18 @@ class HTTPServer(Server): "namespace": "moon" }, ] pod = self.driver.load_pod(data, api_client, ext_client, expose=True) - LOG.debug('wrapper pod={}'.format(pod)) + logger.debug('wrapper pod={}'.format(pod)) - def create_security_function(self, keystone_project_id, - pdp_id, policy_ids, manager_data={}, - active_context=None, - active_context_name=None): + def create_pipeline(self, keystone_project_id, + pdp_id, policy_ids, manager_data=None, + active_context=None, + active_context_name=None): """ Create security functions - :param policy_id: the policy ID mapped to this security function + :param keystone_project_id: the Keystone project id + :param pdp_id: the PDP ID mapped to this pipeline + :param policy_ids: the policy IDs mapped to this pipeline + :param manager_data: data needed to create pods :param active_context: if present, add the security function in this context :param active_context_name: if present, add the security function in @@ -206,11 +207,13 @@ class HTTPServer(Server): security function in all context (ie, in all slaves) :return: None """ + if not manager_data: + manager_data = dict() for key, value in self.driver.get_pods().items(): for _pod in value: if _pod.get('keystone_project_id') == keystone_project_id: - LOG.warning("A pod for this Keystone project {} " - "already exists.".format(keystone_project_id)) + logger.warning("A pod for this Keystone project {} " + "already exists.".format(keystone_project_id)) return plugins = configuration.get_plugins() @@ -231,21 +234,21 @@ class HTTPServer(Server): "namespace": "moon" }, ] - LOG.info("data={}".format(data)) + logger.debug("data={}".format(data)) policies = manager_data.get('policies') if not policies: - LOG.info("No policy data from Manager, trying to get them") + logger.info("No policy data from Manager, trying to get them") policies = requests.get("http://{}:{}/policies".format( self.manager_hostname, self.manager_port)).json().get( "policies", dict()) - LOG.info("policies={}".format(policies)) + logger.debug("policies={}".format(policies)) models = manager_data.get('models') if not models: - LOG.info("No models data from Manager, trying to get them") + logger.info("No models data from Manager, trying to get them") models = requests.get("http://{}:{}/models".format( self.manager_hostname, self.manager_port)).json().get( "models", dict()) - LOG.info("models={}".format(models)) + logger.debug("models={}".format(models)) for policy_id in policy_ids: if policy_id in policies: @@ -263,10 +266,10 @@ class HTTPServer(Server): 'keystone_project_id': keystone_project_id, "namespace": "moon" }) - LOG.info("data={}".format(data)) + logger.debug("data={}".format(data)) contexts, _active_context = self.driver.get_slaves() - LOG.info("active_context_name={}".format(active_context_name)) - LOG.info("active_context={}".format(active_context)) + logger.debug("active_context_name={}".format(active_context_name)) + logger.debug("active_context={}".format(active_context)) if active_context_name: for _context in contexts: if _context["name"] == active_context_name: @@ -276,15 +279,15 @@ class HTTPServer(Server): active_context = _active_context _config = config.new_client_from_config( context=active_context['name']) - LOG.debug("_config={}".format(_config)) + logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) self.driver.load_pod(data, api_client, ext_client, expose=False) return - LOG.info("contexts={}".format(contexts)) + logger.debug("contexts={}".format(contexts)) for _ctx in contexts: _config = config.new_client_from_config(context=_ctx['name']) - LOG.debug("_config={}".format(_config)) + logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) self.driver.load_pod(data, api_client, ext_client, expose=False) diff --git a/moon_orchestrator/moon_orchestrator/server.py b/moon_orchestrator/moon_orchestrator/server.py index ea1a0fbc..88d56e3a 100644 --- a/moon_orchestrator/moon_orchestrator/server.py +++ b/moon_orchestrator/moon_orchestrator/server.py @@ -3,31 +3,37 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -import os import logging from python_moonutilities import configuration, exceptions from moon_orchestrator.http_server import HTTPServer -LOG = logging.getLogger("moon.orchestrator.server") -DOMAIN = "moon_orchestrator" +logger = logging.getLogger("moon.orchestrator.server") -def main(): +def create_server(): configuration.init_logging() try: conf = configuration.get_configuration("components/orchestrator") - hostname = conf["components/orchestrator"].get("hostname", "orchestrator") + hostname = conf["components/orchestrator"].get("hostname", + "orchestrator") port = conf["components/orchestrator"].get("port", 80) bind = conf["components/orchestrator"].get("bind", "127.0.0.1") except exceptions.ConsulComponentNotFound: hostname = "orchestrator" bind = "127.0.0.1" port = 80 - configuration.add_component(uuid="orchestrator", name=hostname, port=port, bind=bind) - LOG.info("Starting server with IP {} on port {} bind to {}".format(hostname, port, bind)) + configuration.add_component(uuid="orchestrator", name=hostname, + port=port, bind=bind) + logger.info("Starting server with IP {} on port {} bind to {}".format( + hostname, port, bind)) return HTTPServer(host=bind, port=port) +def run(): + server = create_server() + server.run() + + if __name__ == '__main__': - server = main() + server = create_server() server.run() diff --git a/moon_orchestrator/requirements.txt b/moon_orchestrator/requirements.txt index 0d952e6c..dbb62043 100644 --- a/moon_orchestrator/requirements.txt +++ b/moon_orchestrator/requirements.txt @@ -3,6 +3,5 @@ flask_restful flask_cors werkzeug python_moonutilities -python_moondb kubernetes pyaml \ No newline at end of file diff --git a/moon_orchestrator/tests/unit_python/test_pods.py b/moon_orchestrator/tests/unit_python/test_pods.py index 42c8404b..0a5a5ba5 100644 --- a/moon_orchestrator/tests/unit_python/test_pods.py +++ b/moon_orchestrator/tests/unit_python/test_pods.py @@ -7,7 +7,7 @@ def test_get_pods(context, monkeypatch): patch_k8s(monkeypatch) import moon_orchestrator.server - server = moon_orchestrator.server.main() + server = moon_orchestrator.server.create_server() _client = server.app.test_client() req = _client.get("/pods") assert req.status_code == 200 @@ -21,7 +21,7 @@ def test_add_pods(context, monkeypatch): patch_k8s(monkeypatch) import moon_orchestrator.server - server = moon_orchestrator.server.main() + server = moon_orchestrator.server.create_server() _client = server.app.test_client() data = { "keystone_project_id": context.get('project_id'), -- cgit 1.2.3-korg