Update to get configuration from the consul

Change-Id: I9455f2415d3b67f26bb5ae2840c329caf779880f

6 files changed, 267 insertions, 27 deletions

diff --git a/moonv4/moon_utilities/Changelog b/moonv4/moon_utilities/Changelog
index bd049b14..ef3d7896 100644
--- a/moonv4/moon_utilities/Changelog
+++ b/moonv4/moon_utilities/Changelog
@@ -21,4 +21,8 @@ CHANGES
 
 1.0.2
 -----
-- Test PyPi upload
\ No newline at end of file
+- Test PyPi upload
+
+1.1.0
+-----
+- Add functions to get configuration from Consul
\ No newline at end of file
diff --git a/moonv4/moon_utilities/moon_utilities/__init__.py b/moonv4/moon_utilities/moon_utilities/__init__.py
index b254b246..2302dea9 100644
--- a/moonv4/moon_utilities/moon_utilities/__init__.py
+++ b/moonv4/moon_utilities/moon_utilities/__init__.py
@@ -3,4 +3,4 @@
 # license which can be found in the file 'LICENSE' in this package distribution
 # or at 'http://www.apache.org/licenses/LICENSE-2.0'.
 
-__version__ = "1.0.2"
+__version__ = "1.1.0"
diff --git a/moonv4/moon_utilities/moon_utilities/configuration.py b/moonv4/moon_utilities/moon_utilities/configuration.py
new file mode 100644
index 00000000..32eeff13
--- /dev/null
+++ b/moonv4/moon_utilities/moon_utilities/configuration.py
@@ -0,0 +1,126 @@
+# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors
+# This software is distributed under the terms and conditions of the 'Apache-2.0'
+# license which can be found in the file 'LICENSE' in this package distribution
+# or at 'http://www.apache.org/licenses/LICENSE-2.0'.
+
+
+import copy
+import base64
+import json
+import requests
+import logging
+import logging.config
+# from oslo_log import log as logging
+from oslo_config import cfg
+# import oslo_messaging
+from moon_utilities import exceptions
+
+LOG = logging.getLogger("moon.utilities")
+
+CONSUL_HOST = "consul"
+CONSUL_PORT = "8500"
+
+DATABASE = "database"
+SLAVE = "slave"
+MESSENGER = "messenger"
+KEYSTONE = "keystone"
+DOCKER = "docker"
+COMPONENTS = "components"
+
+
+def init_logging():
+    config = get_configuration("logging")
+    logging.config.dictConfig(config['logging'])
+
+
+def init_oslo_config():
+    cfg.CONF.transport_url = get_configuration("messenger")['messenger']['url']
+    cfg.CONF.rpc_response_timeout = 5
+
+
+def increment_port():
+    components_port_start = int(get_configuration("components_port_start")['components_port_start'])
+    components_port_start += 1
+    url = "http://{}:{}/v1/kv/components_port_start".format(CONSUL_HOST, CONSUL_PORT)
+    req = requests.put(url, json=str(components_port_start))
+    if req.status_code != 200:
+        LOG.info("url={}".format(url))
+        raise exceptions.ConsulError
+    return components_port_start
+
+
+def get_configuration(key):
+    url = "http://{}:{}/v1/kv/{}".format(CONSUL_HOST, CONSUL_PORT, key)
+    req = requests.get(url)
+    if req.status_code != 200:
+        LOG.info("url={}".format(url))
+        raise exceptions.ConsulComponentNotFound
+    data = req.json()
+    if len(data) == 1:
+        data = data[0]
+        return {data["Key"]: json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+    else:
+        return [
+            {item["Key"]: json.loads(base64.b64decode(item["Value"]).decode("utf-8"))}
+            for item in data
+        ]
+
+
+def add_component(name, uuid, port=None, bind="127.0.0.1", keystone_id="", extra=None, container=None):
+    data = {
+        "hostname": name,
+        "keystone_id": keystone_id,
+        "bind": bind,
+        "port": port,
+        "extra": extra,
+        "container": container
+    }
+    req = requests.put(
+        "http://{}:{}/v1/kv/components/{}".format(CONSUL_HOST, CONSUL_PORT, uuid),
+        headers={"content-type": "application/json"},
+        json=data
+    )
+    if req.status_code != 200:
+        LOG.debug("url={}".format("http://{}:{}/v1/kv/components/{}".format(CONSUL_HOST, CONSUL_PORT, uuid)))
+        LOG.debug("data={}".format(data))
+        raise exceptions.ConsulError
+    LOG.info("Add component {}".format(req.text))
+    return get_configuration("components/"+uuid)
+
+
+def get_plugins():
+    url = "http://{}:{}/v1/kv/plugins?recurse=true".format(CONSUL_HOST, CONSUL_PORT)
+    req = requests.get(url)
+    if req.status_code != 200:
+        LOG.info("url={}".format(url))
+        raise exceptions.ConsulError
+    data = req.json()
+    if len(data) == 1:
+        data = data[0]
+        return {data["Key"].replace("plugins/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+    else:
+        return {
+            item["Key"].replace("plugins/", ""): json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+            for item in data
+        }
+
+
+def get_components():
+    url = "http://{}:{}/v1/kv/components?recurse=true".format(CONSUL_HOST, CONSUL_PORT)
+    req = requests.get(url)
+    if req.status_code != 200:
+        LOG.info("url={}".format(url))
+        raise exceptions.ConsulError
+    data = req.json()
+    if len(data) == 1:
+        data = data[0]
+        return {data["Key"].replace("components/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+    else:
+        return {
+            item["Key"].replace("components/", ""): json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+            for item in data
+        }
+
+
+init_logging()
+init_oslo_config()
diff --git a/moonv4/moon_utilities/moon_utilities/exceptions.py b/moonv4/moon_utilities/moon_utilities/exceptions.py
index f642fb57..ba5ecf46 100644
--- a/moonv4/moon_utilities/moon_utilities/exceptions.py
+++ b/moonv4/moon_utilities/moon_utilities/exceptions.py
@@ -5,7 +5,7 @@
 
 from oslo_log import log as logging
 from werkzeug.exceptions import HTTPException
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger("moon.utilities.exceptions")
 _ = str
 
 
@@ -475,6 +475,9 @@ class RuleUnknown(AdminRule):
     logger = "ERROR"
 
 
+# Keystone exceptions
+
+
 class KeystoneError(MoonError):
     description = _("There is an error connecting to Keystone.")
     code = 400
@@ -503,3 +506,36 @@ class KeystoneUserConflict(KeystoneUserError):
     logger = "ERROR"
 
 
+# Consul exceptions
+
+
+class ConsulError(MoonError):
+    description = _("There is an error connecting to Consul.")
+    code = 400
+    title = 'Consul error'
+    logger = "ERROR"
+
+
+class ConsulComponentNotFound(ConsulError):
+    description = _("The component do not exist in Consul database.")
+    code = 500
+    title = 'Consul error'
+    logger = "WARNING"
+
+
+# Containers exceptions
+
+
+class DockerError(MoonError):
+    description = _("There is an error with Docker.")
+    code = 400
+    title = 'Docker error'
+    logger = "ERROR"
+
+
+class ContainerMissing(DockerError):
+    description = _("Some containers are missing.")
+    code = 400
+    title = 'Container missing'
+    logger = "ERROR"
+
diff --git a/moonv4/moon_utilities/moon_utilities/security_functions.py b/moonv4/moon_utilities/moon_utilities/security_functions.py
index 57baeebb..ad1a44fa 100644
--- a/moonv4/moon_utilities/moon_utilities/security_functions.py
+++ b/moonv4/moon_utilities/moon_utilities/security_functions.py
@@ -6,16 +6,36 @@
 
 import copy
 import re
+import os
 import types
 import requests
+import time
+from functools import wraps
+from flask import request
 from oslo_log import log as logging
 from oslo_config import cfg
 import oslo_messaging
 from moon_utilities import exceptions
+from moon_utilities import configuration
 
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger("moon.utilities." + __name__)
 CONF = cfg.CONF
 
+keystone_config = configuration.get_configuration("openstack/keystone")["openstack/keystone"]
+slave = configuration.get_configuration(configuration.SLAVE)["slave"]
+
+__transport_master = oslo_messaging.get_transport(cfg.CONF, slave.get("master_url"))
+__transport = oslo_messaging.get_transport(CONF)
+
+__n_transport = oslo_messaging.get_notification_transport(CONF)
+__n_notifier = oslo_messaging.Notifier(__n_transport,
+                                       'router.host',
+                                       driver='messagingv2',
+                                       topics=['authz-workers'])
+__n_notifier = __n_notifier.prepare(publisher_id='router')
+
+__targets = {}
+
 
 def filter_input(func_or_str):
 
@@ -92,15 +112,15 @@ def enforce(action_names, object_name, **extra):
 
 def login(user=None, password=None, domain=None, project=None, url=None):
     if not user:
-        user = CONF.keystone.user
+        user = keystone_config['user']
     if not password:
-        password = CONF.keystone.password
+        password = keystone_config['password']
     if not domain:
-        domain = CONF.keystone.domain
+        domain = keystone_config['domain']
     if not project:
-        project = CONF.keystone.project
+        project = keystone_config['project']
     if not url:
-        url = CONF.keystone.url
+        url = keystone_config['url']
     headers = {
         "Content-Type": "application/json"
     }
@@ -133,7 +153,7 @@ def login(user=None, password=None, domain=None, project=None, url=None):
 
     req = requests.post("{}/auth/tokens".format(url),
                         json=data_auth, headers=headers,
-                        verify=CONF.keystone.server_crt)
+                        verify=keystone_config['certificate'])
 
     if req.status_code in (200, 201, 204):
         headers['X-Auth-Token'] = req.headers['X-Subject-Token']
@@ -144,26 +164,14 @@ def login(user=None, password=None, domain=None, project=None, url=None):
 
 def logout(headers, url=None):
     if not url:
-        url = CONF.keystone.url
+        url = keystone_config['url']
     headers['X-Subject-Token'] = headers['X-Auth-Token']
-    req = requests.delete("{}/auth/tokens".format(url), headers=headers, verify=CONF.keystone.server_crt)
+    req = requests.delete("{}/auth/tokens".format(url), headers=headers, verify=keystone_config['certificate'])
     if req.status_code in (200, 201, 204):
         return
     LOG.error(req.text)
     raise exceptions.KeystoneError
 
-__transport_master = oslo_messaging.get_transport(cfg.CONF, CONF.slave.master_url)
-__transport = oslo_messaging.get_transport(CONF)
-
-__n_transport = oslo_messaging.get_notification_transport(CONF)
-__n_notifier = oslo_messaging.Notifier(__n_transport,
-                                       'router.host',
-                                       driver='messagingv2',
-                                       topics=['authz-workers'])
-__n_notifier = __n_notifier.prepare(publisher_id='router')
-
-__targets = {}
-
 
 def notify(request_id, container_id, payload, event_type="authz"):
     ctxt = {
@@ -176,7 +184,7 @@ def notify(request_id, container_id, payload, event_type="authz"):
     __n_notifier.critical(ctxt, event_type, payload=payload)
 
 
-def call(endpoint, ctx=None, method="route", **kwargs):
+def call(endpoint="security_router", ctx=None, method="route", **kwargs):
     if not ctx:
         ctx = dict()
     if endpoint not in __targets:
@@ -187,13 +195,14 @@ def call(endpoint, ctx=None, method="route", **kwargs):
                                                                              __targets[endpoint]["endpoint"])
         __targets[endpoint]["client"]["external"] = oslo_messaging.RPCClient(__transport_master,
                                                                              __targets[endpoint]["endpoint"])
-    if 'call_master' in ctx and ctx['call_master'] and CONF.slave.master_url:
+    if 'call_master' in ctx and ctx['call_master'] and slave.get("master_url"):
         client = __targets[endpoint]["client"]["external"]
         LOG.info("Calling master {} on {}...".format(method, endpoint))
     else:
         client = __targets[endpoint]["client"]["internal"]
         LOG.info("Calling {} on {}...".format(method, endpoint))
     result = copy.deepcopy(client.call(ctx, method, **kwargs))
+    LOG.info("result={}".format(result))
     del client
     return result
 
@@ -429,3 +438,67 @@ pdp_set: {pdp_set}
     @pdp_set.deleter
     def pdp_set(self):
         self.__pdp_set = {}
+
+TOKENS = {}
+
+
+def check_token(token, url=None):
+    _verify = False
+    if keystone_config['certificate']:
+        _verify = keystone_config['certificate']
+    try:
+        os.environ.pop("http_proxy")
+        os.environ.pop("https_proxy")
+    except KeyError:
+        pass
+    if not url:
+        url = keystone_config['url']
+    headers = {
+        "Content-Type": "application/json",
+        'X-Subject-Token': token,
+        'X-Auth-Token': token,
+    }
+    if not keystone_config['check_token']:
+        # TODO (asteroide): must send the admin id
+        return "admin" if not token else token
+    elif keystone_config['check_token'].lower() in ("false", "no", "n"):
+        # TODO (asteroide): must send the admin id
+        return "admin" if not token else token
+    if keystone_config['check_token'].lower() in ("yes", "y", "true"):
+        if token in TOKENS:
+            delta = time.mktime(TOKENS[token]["expires_at"]) - time.mktime(time.gmtime())
+            if delta > 0:
+                return TOKENS[token]["user"]
+            raise exceptions.KeystoneError
+        else:
+            req = requests.get("{}/auth/tokens".format(url), headers=headers, verify=_verify)
+            if req.status_code in (200, 201):
+                # Note (asteroide): the time stamps is not in ISO 8601, so it is necessary to delete
+                # characters after the dot
+                token_time = req.json().get("token").get("expires_at").split(".")
+                TOKENS[token] = dict()
+                TOKENS[token]["expires_at"] = time.strptime(token_time[0], "%Y-%m-%dT%H:%M:%S")
+                TOKENS[token]["user"] = req.json().get("token").get("user").get("id")
+                return TOKENS[token]["user"]
+            LOG.error("{} - {}".format(req.status_code, req.text))
+            raise exceptions.KeystoneError
+    elif keystone_config['check_token'].lower() == "strict":
+        req = requests.head("{}/auth/tokens".format(url), headers=headers, verify=_verify)
+        if req.status_code in (200, 201):
+            return token
+        LOG.error("{} - {}".format(req.status_code, req.text))
+        raise exceptions.KeystoneError
+    raise exceptions.KeystoneError
+
+
+def check_auth(function):
+    @wraps(function)
+    def wrapper(*args, **kwargs):
+        token = request.headers.get('X-Auth-Token')
+        token = check_token(token)
+        if not token:
+            raise exceptions.AuthException
+        user_id = kwargs.pop("user_id", token)
+        result = function(*args, **kwargs, user_id=user_id)
+        return result
+    return wrapper
diff --git a/moonv4/moon_utilities/requirements.txt b/moonv4/moon_utilities/requirements.txt
index c569e00b..eeb36ad0 100644
--- a/moonv4/moon_utilities/requirements.txt
+++ b/moonv4/moon_utilities/requirements.txt
@@ -3,4 +3,5 @@ oslo.messaging
 oslo.config
 oslo.log
 vine
-werkzeug
\ No newline at end of file
+werkzeug
+flask
\ No newline at end of file