adding comments for the cases should be handled for configuration module

refactoring url of component port to match same pattern

integrating request wrapper

adding validation for the key in the object before accessing it

update some conditions
uncomment required test cases after validation done

Change-Id: I5f0e22ca4fd803992c051ec238f91ed0703cd1a2
Signed-off-by: sgdt6900 <rhanafy.ext@orange.com>

5 files changed, 50 insertions, 29 deletions

diff --git a/python_moonutilities/python_moonutilities/configuration.py b/python_moonutilities/python_moonutilities/configuration.py
index 51587582..9a044db7 100644
--- a/python_moonutilities/python_moonutilities/configuration.py
+++ b/python_moonutilities/python_moonutilities/configuration.py
@@ -6,7 +6,7 @@
 
 import base64
 import json
-import requests
+import python_moonutilities.request_wrapper as requests
 import logging.config
 from python_moonutilities import exceptions
 
@@ -25,18 +25,20 @@ def init_logging():
     config = get_configuration("logging")
     logging.config.dictConfig(config['logging'])
 
-
 def increment_port():
-    components_port_start = int(get_configuration("components_port_start")['components_port_start'])
-    components_port_start += 1
-    url = "http://{}:{}/v1/kv/components_port_start".format(CONSUL_HOST, CONSUL_PORT)
+    components_object = get_configuration("components/port_start")
+    if 'port_start' in components_object:
+        components_port_start = int(get_configuration("components/port_start")['port_start'])
+        components_port_start += 1
+    else:
+        raise exceptions.ConsulComponentContentError("error={}".format(components_object))
+    url = "http://{}:{}/v1/kv/components/port_start".format(CONSUL_HOST, CONSUL_PORT)
     req = requests.put(url, json=str(components_port_start))
     if req.status_code != 200:
         logger.info("url={}".format(url))
         raise exceptions.ConsulError
     return components_port_start
 
-
 def get_configuration(key):
     url = "http://{}:{}/v1/kv/{}".format(CONSUL_HOST, CONSUL_PORT, key)
     req = requests.get(url)
@@ -46,14 +48,17 @@ def get_configuration(key):
     data = req.json()
     if len(data) == 1:
         data = data[0]
-        return {data["Key"]: json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+        if all( k in data for k in ("Key", "Value")) :
+            return {data["Key"]: json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+        raise exceptions.ConsulComponentContentError("error={}".format(data))
     else:
         return [
-            {item["Key"]: json.loads(base64.b64decode(item["Value"]).decode("utf-8"))}
-            for item in data
+            {
+                item["Key"]: json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+                if all(k in item for k in ("Key", "Value")) else logger.warning("invalidate content {}".format(item))
+            } for item in data
         ]
 
-
 def add_component(name, uuid, port=None, bind="127.0.0.1", keystone_id="", extra=None, container=None):
     data = {
         "hostname": name,
@@ -75,7 +80,6 @@ def add_component(name, uuid, port=None, bind="127.0.0.1", keystone_id="", extra
     logger.info("Add component {}".format(req.text))
     return configuration.get_configuration("components/"+uuid)
 
-
 def get_plugins():
     url = "http://{}:{}/v1/kv/plugins?recurse=true".format(CONSUL_HOST, CONSUL_PORT)
     req = requests.get(url)
@@ -85,14 +89,16 @@ def get_plugins():
     data = req.json()
     if len(data) == 1:
         data = data[0]
-        return {data["Key"].replace("plugins/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+        if all(k in data for k in ("Key", "Value")):
+            return {data["Key"].replace("plugins/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+        raise exceptions.ConsulComponentContentError("error={}".format(data))
     else:
         return {
             item["Key"].replace("plugins/", ""): json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+            if all(k in item for k in ("Key", "Value")) else logger.warning("invalidate content {}".format(item))
             for item in data
         }
 
-
 def get_components():
     url = "http://{}:{}/v1/kv/components?recurse=true".format(CONSUL_HOST, CONSUL_PORT)
     req = requests.get(url)
@@ -102,10 +108,13 @@ def get_components():
     data = req.json()
     if len(data) == 1:
         data = data[0]
-        return {data["Key"].replace("components/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+        if all(k in data for k in ("Key", "Value")):
+            return {data["Key"].replace("components/", ""): json.loads(base64.b64decode(data["Value"]).decode("utf-8"))}
+        raise exceptions.ConsulComponentContentError("error={}".format(data))
     else:
         return {
             item["Key"].replace("components/", ""): json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+            if all(k in item for k in ("Key", "Value")) else logger.warning("invalidate content {}".format(item))
             for item in data
         }
 
diff --git a/python_moonutilities/python_moonutilities/exceptions.py b/python_moonutilities/python_moonutilities/exceptions.py
index f14d6abf..8bcca72e 100644
--- a/python_moonutilities/python_moonutilities/exceptions.py
+++ b/python_moonutilities/python_moonutilities/exceptions.py
@@ -504,6 +504,11 @@ class ConsulComponentNotFound(ConsulError):
     title = 'Consul error'
     logger = "WARNING"
 
+class ConsulComponentContentError(ConsulError):
+    description = _("invalid content of component .")
+    code = 500
+    title = 'Consul error'
+    logger = "WARNING"
 
 # Containers exceptions
 
diff --git a/python_moonutilities/python_moonutilities/request_wrapper.py b/python_moonutilities/python_moonutilities/request_wrapper.py
index 8cf5b997..f1603b9d 100644
--- a/python_moonutilities/python_moonutilities/request_wrapper.py
+++ b/python_moonutilities/python_moonutilities/request_wrapper.py
@@ -9,4 +9,14 @@ def get(url):
         raise exceptions.ConsulError("request failure ",e)
     except:
         raise exceptions.ConsulError("Unexpected error ", sys.exc_info()[0])
+    return response
+
+
+def put(url, json=""):
+    try:
+        response = requests.put(url,json=json)
+    except requests.exceptions.RequestException as e:
+        raise exceptions.ConsulError("request failure ",e)
+    except:
+        raise exceptions.ConsulError("Unexpected error ", sys.exc_info()[0])
     return response
\ No newline at end of file
diff --git a/python_moonutilities/tests/unit_python/mock_repo/urls.py b/python_moonutilities/tests/unit_python/mock_repo/urls.py
index a5b1e63b..75b55927 100644
--- a/python_moonutilities/tests/unit_python/mock_repo/urls.py
+++ b/python_moonutilities/tests/unit_python/mock_repo/urls.py
@@ -9,11 +9,11 @@ def register_components(m):
             json=[{'Key': component, 'Value': comp_util.get_b64_conf(component)}]
         )
     m.register_uri(
-        'GET', 'http://consul:8500/v1/kv/components_port_start',
-        json=[{'Key': 'components_port_start', 'Value': comp_util.get_b64_conf("components/port_start")}]
+        'GET', 'http://consul:8500/v1/kv/components/port_start',
+        json=[{'Key': 'port_start', 'Value': comp_util.get_b64_conf("components/port_start")}]
     )
     m.register_uri(
-        'PUT', 'http://consul:8500/v1/kv/components_port_start',
+        'PUT', 'http://consul:8500/v1/kv/components/port_start',
         json=[]
     )
 
diff --git a/python_moonutilities/tests/unit_python/test_configuration.py b/python_moonutilities/tests/unit_python/test_configuration.py
index 10618f1c..8ca389bf 100644
--- a/python_moonutilities/tests/unit_python/test_configuration.py
+++ b/python_moonutilities/tests/unit_python/test_configuration.py
@@ -5,7 +5,7 @@ import requests_mock
 
 def test_get_configuration_success():
     from python_moonutilities import configuration
-    assert configuration.get_configuration("components/port_start")["components/port_start"] == comp_util.CONF["components"]["port_start"]
+    assert configuration.get_configuration("components/port_start")["port_start"] == comp_util.CONF["components"]["port_start"]
 
 
 @requests_mock.Mocker(kw='mock')
@@ -18,27 +18,24 @@ def test_get_configuration_not_found(**kwargs):
         configuration.get_configuration("components/port_start_wrong")
     assert str(exception_info.value) == '500: Consul error'
 
-
-# [TODO] this test used to test the invalid response
-# it should be un commented and run after refactoring the related part
 @requests_mock.Mocker(kw='mock')
 def test_get_configuration_invalid_response(**kwargs):
     from python_moonutilities import configuration
 
-    kwargs['mock'].get('http://consul:8500/v1/kv/components_port_start', json=[
-        {"components_port_start":'components_port_start', 'Value': comp_util.get_b64_conf("components/port_start")}
+    kwargs['mock'].get('http://consul:8500/v1/kv/components/port_start', json=[
+        {"port_start":'port_start', 'Value': comp_util.get_b64_conf("components/port_start")}
     ])
-    # with pytest.raises(Exception) as exception_info:
-    # configuration.get_configuration("components_port_start")
-    # assert str(exception_info.value) == '500: Consul error'
+    with pytest.raises(Exception) as exception_info:
+        configuration.get_configuration("components/port_start")
+    assert str(exception_info.value) == '500: Consul error'
 
 
 @requests_mock.Mocker(kw='mock')
 def test_put_increment_port_failure(**kwargs):
     from python_moonutilities import configuration
-    kwargs['mock'].put('http://consul:8500/v1/kv/components_port_start', json=[], status_code=400)
-    kwargs['mock'].get('http://consul:8500/v1/kv/components_port_start', json=[
-        {'Key': 'components_port_start', 'Value': comp_util.get_b64_conf("components/port_start")}
+    kwargs['mock'].put('http://consul:8500/v1/kv/components/port_start', json=[], status_code=400)
+    kwargs['mock'].get('http://consul:8500/v1/kv/components/port_start', json=[
+        {'Key': 'port_start', 'Value': comp_util.get_b64_conf("components/port_start")}
     ], status_code=200)
     with pytest.raises(Exception) as exception_info:
         configuration.increment_port()