From 025de97a3ba7ed570adfcadbfadadc14d7e57428 Mon Sep 17 00:00:00 2001 From: asteroide Date: Tue, 25 Jul 2017 17:39:17 +0200 Subject: Update to get configuration from the consul Change-Id: I9455f2415d3b67f26bb5ae2840c329caf779880f --- moonv4/moon_utilities/Changelog | 6 +- moonv4/moon_utilities/moon_utilities/__init__.py | 2 +- .../moon_utilities/moon_utilities/configuration.py | 126 +++++++++++++++++++++ moonv4/moon_utilities/moon_utilities/exceptions.py | 38 ++++++- .../moon_utilities/security_functions.py | 119 +++++++++++++++---- moonv4/moon_utilities/requirements.txt | 3 +- 6 files changed, 267 insertions(+), 27 deletions(-) create mode 100644 moonv4/moon_utilities/moon_utilities/configuration.py (limited to 'moonv4/moon_utilities') diff --git a/moonv4/moon_utilities/Changelog b/moonv4/moon_utilities/Changelog index bd049b14..ef3d7896 100644 --- a/moonv4/moon_utilities/Changelog +++ b/moonv4/moon_utilities/Changelog @@ -21,4 +21,8 @@ CHANGES 1.0.2 ----- -- Test PyPi upload \ No newline at end of file +- Test PyPi upload + +1.1.0 +----- +- Add functions to get configuration from Consul \ No newline at end of file diff --git a/moonv4/moon_utilities/moon_utilities/__init__.py b/moonv4/moon_utilities/moon_utilities/__init__.py index b254b246..2302dea9 100644 --- a/moonv4/moon_utilities/moon_utilities/__init__.py +++ b/moonv4/moon_utilities/moon_utilities/__init__.py @@ -3,4 +3,4 @@ # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. -__version__ = "1.0.2" +__version__ = "1.1.0" diff --git a/moonv4/moon_utilities/moon_utilities/configuration.py b/moonv4/moon_utilities/moon_utilities/configuration.py new file mode 100644 index 00000000..32eeff13 --- /dev/null +++ b/moonv4/moon_utilities/moon_utilities/configuration.py @@ -0,0 +1,126 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + + +import copy +import base64 +import json +import requests +import logging +import logging.config +# from oslo_log import log as logging +from oslo_config import cfg +# import oslo_messaging +from moon_utilities import exceptions + +LOG = logging.getLogger("moon.utilities") + +CONSUL_HOST = "consul" +CONSUL_PORT = "8500" + +DATABASE = "database" +SLAVE = "slave" +MESSENGER = "messenger" +KEYSTONE = "keystone" +DOCKER = "docker" +COMPONENTS = "components" + + +def init_logging(): + config = get_configuration("logging") + logging.config.dictConfig(config['logging']) + + +def init_oslo_config(): + cfg.CONF.transport_url = get_configuration("messenger")['messenger']['url'] + cfg.CONF.rpc_response_timeout = 5 + + +def increment_port(): + components_port_start = int(get_configuration("components_port_start")['components_port_start']) + components_port_start += 1 + url = "http://{}:{}/v1/kv/components_port_start".format(CONSUL_HOST, CONSUL_PORT) + req = requests.put(url, json=str(components_port_start)) + if req.status_code != 200: + LOG.info("url={}".format(url)) + raise exceptions.ConsulError + return components_port_start + + +def get_configuration(key): + url = "http://{}:{}/v1/kv/{}".format(CONSUL_HOST, CONSUL_PORT, key) + req = requests.get(url) + if req.status_code != 200: + LOG.info("url={}".format(url)) + raise exceptions.ConsulComponentNotFound + data = req.json() + if len(data) == 1: + data = data[0] + return {data["Key"]: json.loads(base64.b64decode(data["Value"]).decode("utf-8"))} + else: + return [ + {item["Key"]: json.loads(base64.b64decode(item["Value"]).decode("utf-8"))} + for item in data + ] + + +def add_component(name, uuid, port=None, bind="127.0.0.1", keystone_id="", extra=None, container=None): + data = { + "hostname": name, + "keystone_id": keystone_id, + "bind": bind, + "port": port, + "extra": extra, + "container": container + } + req = requests.put( + "http://{}:{}/v1/kv/components/{}".format(CONSUL_HOST, CONSUL_PORT, uuid), + headers={"content-type": "application/json"}, + json=data + ) + if req.status_code != 200: + LOG.debug("url={}".format("http://{}:{}/v1/kv/components/{}".format(CONSUL_HOST, CONSUL_PORT, uuid))) + LOG.debug("data={}".format(data)) + raise exceptions.ConsulError + LOG.info("Add component {}".format(req.text)) + return get_configuration("components/"+uuid) + + +def get_plugins(): + url = "http://{}:{}/v1/kv/plugins?recurse=true".format(CONSUL_HOST, CONSUL_PORT) + req = requests.get(url) + if req.status_code != 200: + LOG.info("url={}".format(url)) + raise exceptions.ConsulError + data = req.json() + if len(data) == 1: + data = data[0] + return {data["Key"].replace("plugins/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))} + else: + return { + item["Key"].replace("plugins/", ""): json.loads(base64.b64decode(item["Value"]).decode("utf-8")) + for item in data + } + + +def get_components(): + url = "http://{}:{}/v1/kv/components?recurse=true".format(CONSUL_HOST, CONSUL_PORT) + req = requests.get(url) + if req.status_code != 200: + LOG.info("url={}".format(url)) + raise exceptions.ConsulError + data = req.json() + if len(data) == 1: + data = data[0] + return {data["Key"].replace("components/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))} + else: + return { + item["Key"].replace("components/", ""): json.loads(base64.b64decode(item["Value"]).decode("utf-8")) + for item in data + } + + +init_logging() +init_oslo_config() diff --git a/moonv4/moon_utilities/moon_utilities/exceptions.py b/moonv4/moon_utilities/moon_utilities/exceptions.py index f642fb57..ba5ecf46 100644 --- a/moonv4/moon_utilities/moon_utilities/exceptions.py +++ b/moonv4/moon_utilities/moon_utilities/exceptions.py @@ -5,7 +5,7 @@ from oslo_log import log as logging from werkzeug.exceptions import HTTPException -LOG = logging.getLogger(__name__) +LOG = logging.getLogger("moon.utilities.exceptions") _ = str @@ -475,6 +475,9 @@ class RuleUnknown(AdminRule): logger = "ERROR" +# Keystone exceptions + + class KeystoneError(MoonError): description = _("There is an error connecting to Keystone.") code = 400 @@ -503,3 +506,36 @@ class KeystoneUserConflict(KeystoneUserError): logger = "ERROR" +# Consul exceptions + + +class ConsulError(MoonError): + description = _("There is an error connecting to Consul.") + code = 400 + title = 'Consul error' + logger = "ERROR" + + +class ConsulComponentNotFound(ConsulError): + description = _("The component do not exist in Consul database.") + code = 500 + title = 'Consul error' + logger = "WARNING" + + +# Containers exceptions + + +class DockerError(MoonError): + description = _("There is an error with Docker.") + code = 400 + title = 'Docker error' + logger = "ERROR" + + +class ContainerMissing(DockerError): + description = _("Some containers are missing.") + code = 400 + title = 'Container missing' + logger = "ERROR" + diff --git a/moonv4/moon_utilities/moon_utilities/security_functions.py b/moonv4/moon_utilities/moon_utilities/security_functions.py index 57baeebb..ad1a44fa 100644 --- a/moonv4/moon_utilities/moon_utilities/security_functions.py +++ b/moonv4/moon_utilities/moon_utilities/security_functions.py @@ -6,16 +6,36 @@ import copy import re +import os import types import requests +import time +from functools import wraps +from flask import request from oslo_log import log as logging from oslo_config import cfg import oslo_messaging from moon_utilities import exceptions +from moon_utilities import configuration -LOG = logging.getLogger(__name__) +LOG = logging.getLogger("moon.utilities." + __name__) CONF = cfg.CONF +keystone_config = configuration.get_configuration("openstack/keystone")["openstack/keystone"] +slave = configuration.get_configuration(configuration.SLAVE)["slave"] + +__transport_master = oslo_messaging.get_transport(cfg.CONF, slave.get("master_url")) +__transport = oslo_messaging.get_transport(CONF) + +__n_transport = oslo_messaging.get_notification_transport(CONF) +__n_notifier = oslo_messaging.Notifier(__n_transport, + 'router.host', + driver='messagingv2', + topics=['authz-workers']) +__n_notifier = __n_notifier.prepare(publisher_id='router') + +__targets = {} + def filter_input(func_or_str): @@ -92,15 +112,15 @@ def enforce(action_names, object_name, **extra): def login(user=None, password=None, domain=None, project=None, url=None): if not user: - user = CONF.keystone.user + user = keystone_config['user'] if not password: - password = CONF.keystone.password + password = keystone_config['password'] if not domain: - domain = CONF.keystone.domain + domain = keystone_config['domain'] if not project: - project = CONF.keystone.project + project = keystone_config['project'] if not url: - url = CONF.keystone.url + url = keystone_config['url'] headers = { "Content-Type": "application/json" } @@ -133,7 +153,7 @@ def login(user=None, password=None, domain=None, project=None, url=None): req = requests.post("{}/auth/tokens".format(url), json=data_auth, headers=headers, - verify=CONF.keystone.server_crt) + verify=keystone_config['certificate']) if req.status_code in (200, 201, 204): headers['X-Auth-Token'] = req.headers['X-Subject-Token'] @@ -144,26 +164,14 @@ def login(user=None, password=None, domain=None, project=None, url=None): def logout(headers, url=None): if not url: - url = CONF.keystone.url + url = keystone_config['url'] headers['X-Subject-Token'] = headers['X-Auth-Token'] - req = requests.delete("{}/auth/tokens".format(url), headers=headers, verify=CONF.keystone.server_crt) + req = requests.delete("{}/auth/tokens".format(url), headers=headers, verify=keystone_config['certificate']) if req.status_code in (200, 201, 204): return LOG.error(req.text) raise exceptions.KeystoneError -__transport_master = oslo_messaging.get_transport(cfg.CONF, CONF.slave.master_url) -__transport = oslo_messaging.get_transport(CONF) - -__n_transport = oslo_messaging.get_notification_transport(CONF) -__n_notifier = oslo_messaging.Notifier(__n_transport, - 'router.host', - driver='messagingv2', - topics=['authz-workers']) -__n_notifier = __n_notifier.prepare(publisher_id='router') - -__targets = {} - def notify(request_id, container_id, payload, event_type="authz"): ctxt = { @@ -176,7 +184,7 @@ def notify(request_id, container_id, payload, event_type="authz"): __n_notifier.critical(ctxt, event_type, payload=payload) -def call(endpoint, ctx=None, method="route", **kwargs): +def call(endpoint="security_router", ctx=None, method="route", **kwargs): if not ctx: ctx = dict() if endpoint not in __targets: @@ -187,13 +195,14 @@ def call(endpoint, ctx=None, method="route", **kwargs): __targets[endpoint]["endpoint"]) __targets[endpoint]["client"]["external"] = oslo_messaging.RPCClient(__transport_master, __targets[endpoint]["endpoint"]) - if 'call_master' in ctx and ctx['call_master'] and CONF.slave.master_url: + if 'call_master' in ctx and ctx['call_master'] and slave.get("master_url"): client = __targets[endpoint]["client"]["external"] LOG.info("Calling master {} on {}...".format(method, endpoint)) else: client = __targets[endpoint]["client"]["internal"] LOG.info("Calling {} on {}...".format(method, endpoint)) result = copy.deepcopy(client.call(ctx, method, **kwargs)) + LOG.info("result={}".format(result)) del client return result @@ -429,3 +438,67 @@ pdp_set: {pdp_set} @pdp_set.deleter def pdp_set(self): self.__pdp_set = {} + +TOKENS = {} + + +def check_token(token, url=None): + _verify = False + if keystone_config['certificate']: + _verify = keystone_config['certificate'] + try: + os.environ.pop("http_proxy") + os.environ.pop("https_proxy") + except KeyError: + pass + if not url: + url = keystone_config['url'] + headers = { + "Content-Type": "application/json", + 'X-Subject-Token': token, + 'X-Auth-Token': token, + } + if not keystone_config['check_token']: + # TODO (asteroide): must send the admin id + return "admin" if not token else token + elif keystone_config['check_token'].lower() in ("false", "no", "n"): + # TODO (asteroide): must send the admin id + return "admin" if not token else token + if keystone_config['check_token'].lower() in ("yes", "y", "true"): + if token in TOKENS: + delta = time.mktime(TOKENS[token]["expires_at"]) - time.mktime(time.gmtime()) + if delta > 0: + return TOKENS[token]["user"] + raise exceptions.KeystoneError + else: + req = requests.get("{}/auth/tokens".format(url), headers=headers, verify=_verify) + if req.status_code in (200, 201): + # Note (asteroide): the time stamps is not in ISO 8601, so it is necessary to delete + # characters after the dot + token_time = req.json().get("token").get("expires_at").split(".") + TOKENS[token] = dict() + TOKENS[token]["expires_at"] = time.strptime(token_time[0], "%Y-%m-%dT%H:%M:%S") + TOKENS[token]["user"] = req.json().get("token").get("user").get("id") + return TOKENS[token]["user"] + LOG.error("{} - {}".format(req.status_code, req.text)) + raise exceptions.KeystoneError + elif keystone_config['check_token'].lower() == "strict": + req = requests.head("{}/auth/tokens".format(url), headers=headers, verify=_verify) + if req.status_code in (200, 201): + return token + LOG.error("{} - {}".format(req.status_code, req.text)) + raise exceptions.KeystoneError + raise exceptions.KeystoneError + + +def check_auth(function): + @wraps(function) + def wrapper(*args, **kwargs): + token = request.headers.get('X-Auth-Token') + token = check_token(token) + if not token: + raise exceptions.AuthException + user_id = kwargs.pop("user_id", token) + result = function(*args, **kwargs, user_id=user_id) + return result + return wrapper diff --git a/moonv4/moon_utilities/requirements.txt b/moonv4/moon_utilities/requirements.txt index c569e00b..eeb36ad0 100644 --- a/moonv4/moon_utilities/requirements.txt +++ b/moonv4/moon_utilities/requirements.txt @@ -3,4 +3,5 @@ oslo.messaging oslo.config oslo.log vine -werkzeug \ No newline at end of file +werkzeug +flask \ No newline at end of file -- cgit 1.2.3-korg