Improved error handling when Energy recorder API is unavailable.

Log verbosity is reduced when API is discovered unavailable.
Avoid next calls to API if unavailable at config loading.

Change-Id: I68d169396335ae3891e4b808062058945fc2eca1
Signed-off-by: Benoit HERARD <benoit.herard@orange.com>

2 files changed, 183 insertions, 109 deletions

diff --git a/functest/energy/energy.py b/functest/energy/energy.py
index 372c1d32..580bc6b0 100644
--- a/functest/energy/energy.py
+++ b/functest/energy/energy.py
@@ -55,9 +55,10 @@ def enable_recording(method):
         try:
             return_value = method(*args)
             finish_session(current_scenario)
-        except Exception:  # pylint: disable=broad-except
+        except Exception as exc:  # pylint: disable=broad-except
+            EnergyRecorder.logger.exception(exc)
             finish_session(current_scenario)
-            raise
+            raise exc
         return return_value
     return wrapper
 
@@ -74,6 +75,9 @@ class EnergyRecorder(object):
     # Default initial step
     INITIAL_STEP = "running"
 
+    # Default connection timeout
+    CONNECTION_TIMOUT = 1
+
     @staticmethod
     def load_config():
         """
@@ -94,10 +98,10 @@ class EnergyRecorder(object):
             assert energy_recorder_uri
             assert environment
 
-            energy_recorder_uri += "/recorders/environment/"
-            energy_recorder_uri += urllib.quote_plus(environment)
+            uri_comp = "/recorders/environment/"
+            uri_comp += urllib.quote_plus(environment)
             EnergyRecorder.logger.debug(
-                "API recorder at: " + energy_recorder_uri)
+                "API recorder at: " + energy_recorder_uri + uri_comp)
 
             # Creds
             user = ft_utils.get_functest_config(
@@ -110,11 +114,25 @@ class EnergyRecorder(object):
             else:
                 energy_recorder_api_auth = None
 
+            try:
+                resp = requests.get(energy_recorder_uri + "/monitoring/ping",
+                                    auth=energy_recorder_api_auth,
+                                    headers={
+                                        'content-type': 'application/json'
+                                    },
+                                    timeout=EnergyRecorder.CONNECTION_TIMOUT)
+                api_available = json.loads(resp.text)["status"] == "OK"
+            except Exception:  # pylint: disable=broad-except
+                EnergyRecorder.logger.error(
+                    "Energy recorder API is not available")
+                api_available = False
             # Final config
             EnergyRecorder.energy_recorder_api = {
-                "uri": energy_recorder_uri,
-                "auth": energy_recorder_api_auth
+                "uri": energy_recorder_uri + uri_comp,
+                "auth": energy_recorder_api_auth,
+                "available": api_available
             }
+        return EnergyRecorder.energy_recorder_api["available"]
 
     @staticmethod
     def submit_scenario(scenario, step):
@@ -126,31 +144,36 @@ class EnergyRecorder(object):
             param step: Step name
             :type step: string
         """
-        return_status = True
         try:
-            EnergyRecorder.logger.debug("Submitting scenario")
+            return_status = True
             # Ensure that connectyvity settings are loaded
-            EnergyRecorder.load_config()
+            if EnergyRecorder.load_config():
+                EnergyRecorder.logger.debug("Submitting scenario")
 
-            # Create API payload
-            payload = {
-                "step": step,
-                "scenario": scenario
-            }
-            # Call API to start energy recording
-            response = requests.post(
-                EnergyRecorder.energy_recorder_api["uri"],
-                data=json.dumps(payload),
-                auth=EnergyRecorder.energy_recorder_api["auth"],
-                headers={
-                    'content-type': 'application/json'
+                # Create API payload
+                payload = {
+                    "step": step,
+                    "scenario": scenario
                 }
-            )
-            if response.status_code != 200:
-                log_msg = "Error while submitting scenario\n{}"
-                log_msg = log_msg.format(response.text)
-                EnergyRecorder.logger.info(log_msg)
-                return_status = False
+                # Call API to start energy recording
+                response = requests.post(
+                    EnergyRecorder.energy_recorder_api["uri"],
+                    data=json.dumps(payload),
+                    auth=EnergyRecorder.energy_recorder_api["auth"],
+                    headers={
+                        'content-type': 'application/json'
+                    },
+                    timeout=EnergyRecorder.CONNECTION_TIMOUT
+                )
+                if response.status_code != 200:
+                    EnergyRecorder.logger.error(
+                        "Error while submitting scenario\n%s",
+                        response.text)
+                    return_status = False
+        except requests.exceptions.ConnectionError:
+            EnergyRecorder.logger.warning(
+                "submit_scenario: Unable to connect energy recorder API")
+            return_status = False
         except Exception:  # pylint: disable=broad-except
             # Default exception handler to ensure that method
             # is safe for caller
@@ -170,11 +193,12 @@ class EnergyRecorder(object):
         """
         return_status = True
         try:
-            EnergyRecorder.logger.debug("Starting recording")
-            return_status = EnergyRecorder.submit_scenario(
-                scenario,
-                EnergyRecorder.INITIAL_STEP
-            )
+            if EnergyRecorder.load_config():
+                EnergyRecorder.logger.debug("Starting recording")
+                return_status = EnergyRecorder.submit_scenario(
+                    scenario,
+                    EnergyRecorder.INITIAL_STEP
+                )
 
         except Exception:  # pylint: disable=broad-except
             # Default exception handler to ensure that method
@@ -188,25 +212,30 @@ class EnergyRecorder(object):
     @staticmethod
     def stop():
         """Stop current recording session."""
-        EnergyRecorder.logger.debug("Stopping recording")
         return_status = True
         try:
             # Ensure that connectyvity settings are loaded
-            EnergyRecorder.load_config()
-
-            # Call API to stop energy recording
-            response = requests.delete(
-                EnergyRecorder.energy_recorder_api["uri"],
-                auth=EnergyRecorder.energy_recorder_api["auth"],
-                headers={
-                    'content-type': 'application/json'
-                }
-            )
-            if response.status_code != 200:
-                log_msg = "Error while stating energy recording session\n{}"
-                log_msg = log_msg.format(response.text)
-                EnergyRecorder.logger.error(log_msg)
-                return_status = False
+            if EnergyRecorder.load_config():
+                EnergyRecorder.logger.debug("Stopping recording")
+
+                # Call API to stop energy recording
+                response = requests.delete(
+                    EnergyRecorder.energy_recorder_api["uri"],
+                    auth=EnergyRecorder.energy_recorder_api["auth"],
+                    headers={
+                        'content-type': 'application/json'
+                    },
+                    timeout=EnergyRecorder.CONNECTION_TIMOUT
+                )
+                if response.status_code != 200:
+                    EnergyRecorder.logger.error(
+                        "Error while stating energy recording session\n%s",
+                        response.text)
+                    return_status = False
+        except requests.exceptions.ConnectionError:
+            EnergyRecorder.logger.warning(
+                "stop: Unable to connect energy recorder API")
+            return_status = False
         except Exception:  # pylint: disable=broad-except
             # Default exception handler to ensure that method
             # is safe for caller
@@ -219,31 +248,36 @@ class EnergyRecorder(object):
     @staticmethod
     def set_step(step):
         """Notify energy recording service of current step of the testcase."""
-        EnergyRecorder.logger.debug("Setting step")
         return_status = True
         try:
             # Ensure that connectyvity settings are loaded
-            EnergyRecorder.load_config()
+            if EnergyRecorder.load_config():
+                EnergyRecorder.logger.debug("Setting step")
 
-            # Create API payload
-            payload = {
-                "step": step,
-            }
-
-            # Call API to define step
-            response = requests.post(
-                EnergyRecorder.energy_recorder_api["uri"] + "/step",
-                data=json.dumps(payload),
-                auth=EnergyRecorder.energy_recorder_api["auth"],
-                headers={
-                    'content-type': 'application/json'
+                # Create API payload
+                payload = {
+                    "step": step,
                 }
-            )
-            if response.status_code != 200:
-                log_msg = "Error while setting current step of testcase\n{}"
-                log_msg = log_msg.format(response.text)
-                EnergyRecorder.logger.error(log_msg)
-                return_status = False
+
+                # Call API to define step
+                response = requests.post(
+                    EnergyRecorder.energy_recorder_api["uri"] + "/step",
+                    data=json.dumps(payload),
+                    auth=EnergyRecorder.energy_recorder_api["auth"],
+                    headers={
+                        'content-type': 'application/json'
+                    },
+                    timeout=EnergyRecorder.CONNECTION_TIMOUT
+                )
+                if response.status_code != 200:
+                    EnergyRecorder.logger.error(
+                        "Error while setting current step of testcase\n%s",
+                        response.text)
+                    return_status = False
+        except requests.exceptions.ConnectionError:
+            EnergyRecorder.logger.warning(
+                "set_step: Unable to connect energy recorder API")
+            return_status = False
         except Exception:  # pylint: disable=broad-except
             # Default exception handler to ensure that method
             # is safe for caller
@@ -256,30 +290,34 @@ class EnergyRecorder(object):
     @staticmethod
     def get_current_scenario():
         """Get current running scenario (if any, None else)."""
-        EnergyRecorder.logger.debug("Getting current scenario")
         return_value = None
         try:
             # Ensure that connectyvity settings are loaded
-            EnergyRecorder.load_config()
-
-            # Call API get running scenario
-            response = requests.get(
-                EnergyRecorder.energy_recorder_api["uri"],
-                auth=EnergyRecorder.energy_recorder_api["auth"]
-            )
-            if response.status_code == 200:
-                return_value = json.loads(response.text)
-            elif response.status_code == 404:
-                log_msg = "No current running scenario at {}"
-                log_msg = log_msg.format(
-                    EnergyRecorder.energy_recorder_api["uri"])
-                EnergyRecorder.logger.error(log_msg)
-                return_value = None
-            else:
-                log_msg = "Error while getting current scenario\n{}"
-                log_msg = log_msg.format(response.text)
-                EnergyRecorder.logger.error(log_msg)
-                return_value = None
+            if EnergyRecorder.load_config():
+                EnergyRecorder.logger.debug("Getting current scenario")
+
+                # Call API get running scenario
+                response = requests.get(
+                    EnergyRecorder.energy_recorder_api["uri"],
+                    auth=EnergyRecorder.energy_recorder_api["auth"],
+                    timeout=EnergyRecorder.CONNECTION_TIMOUT
+                )
+                if response.status_code == 200:
+                    return_value = json.loads(response.text)
+                elif response.status_code == 404:
+                    EnergyRecorder.logger.info(
+                        "No current running scenario at %s",
+                        EnergyRecorder.energy_recorder_api["uri"])
+                    return_value = None
+                else:
+                    EnergyRecorder.logger.error(
+                        "Error while getting current scenario\n%s",
+                        response.text)
+                    return_value = None
+        except requests.exceptions.ConnectionError:
+            EnergyRecorder.logger.warning(
+                "get_currernt_sceario: Unable to connect energy recorder API")
+            return_value = None
         except Exception:  # pylint: disable=broad-except
             # Default exception handler to ensure that method
             # is safe for caller
diff --git a/functest/tests/unit/energy/test_functest_energy.py b/functest/tests/unit/energy/test_functest_energy.py
index f8bb13c9..a576e2c3 100644
--- a/functest/tests/unit/energy/test_functest_energy.py
+++ b/functest/tests/unit/energy/test_functest_energy.py
@@ -35,6 +35,15 @@ class MockHttpResponse(object):  # pylint: disable=too-few-public-methods
         self.status_code = status_code
 
 
+API_OK = MockHttpResponse(
+    '{"status": "OK"}',
+    200
+)
+API_KO = MockHttpResponse(
+    '{"message": "API-KO"}',
+    500
+)
+
 RECORDER_OK = MockHttpResponse(
     '{"environment": "UNIT_TEST",'
     ' "step": "string",'
@@ -81,7 +90,7 @@ class EnergyRecorderTest(unittest.TestCase):
 
     @mock.patch('functest.energy.energy.requests.post',
                 return_value=RECORDER_OK)
-    def test_start(self, post_mock=None):
+    def test_start(self, post_mock=None, get_mock=None):
         """EnergyRecorder.start method (regular case)."""
         self.test_load_config()
         self.assertTrue(EnergyRecorder.start(self.case_name))
@@ -89,7 +98,8 @@ class EnergyRecorderTest(unittest.TestCase):
             EnergyRecorder.energy_recorder_api["uri"],
             auth=EnergyRecorder.energy_recorder_api["auth"],
             data=mock.ANY,
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @mock.patch('functest.energy.energy.requests.post',
@@ -102,7 +112,8 @@ class EnergyRecorderTest(unittest.TestCase):
             EnergyRecorder.energy_recorder_api["uri"],
             auth=EnergyRecorder.energy_recorder_api["auth"],
             data=mock.ANY,
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @mock.patch('functest.energy.energy.requests.post',
@@ -115,7 +126,8 @@ class EnergyRecorderTest(unittest.TestCase):
             EnergyRecorder.energy_recorder_api["uri"],
             auth=EnergyRecorder.energy_recorder_api["auth"],
             data=mock.ANY,
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @mock.patch('functest.energy.energy.requests.post',
@@ -128,7 +140,8 @@ class EnergyRecorderTest(unittest.TestCase):
             EnergyRecorder.energy_recorder_api["uri"] + "/step",
             auth=EnergyRecorder.energy_recorder_api["auth"],
             data=mock.ANY,
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @mock.patch('functest.energy.energy.requests.post',
@@ -141,7 +154,8 @@ class EnergyRecorderTest(unittest.TestCase):
             EnergyRecorder.energy_recorder_api["uri"] + "/step",
             auth=EnergyRecorder.energy_recorder_api["auth"],
             data=mock.ANY,
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @mock.patch('functest.energy.energy.requests.post',
@@ -154,7 +168,8 @@ class EnergyRecorderTest(unittest.TestCase):
             EnergyRecorder.energy_recorder_api["uri"] + "/step",
             auth=EnergyRecorder.energy_recorder_api["auth"],
             data=mock.ANY,
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @mock.patch('functest.energy.energy.requests.delete',
@@ -166,7 +181,8 @@ class EnergyRecorderTest(unittest.TestCase):
         delete_mock.assert_called_once_with(
             EnergyRecorder.energy_recorder_api["uri"],
             auth=EnergyRecorder.energy_recorder_api["auth"],
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @mock.patch('functest.energy.energy.requests.delete',
@@ -178,7 +194,8 @@ class EnergyRecorderTest(unittest.TestCase):
         delete_mock.assert_called_once_with(
             EnergyRecorder.energy_recorder_api["uri"],
             auth=EnergyRecorder.energy_recorder_api["auth"],
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @mock.patch('functest.energy.energy.requests.delete',
@@ -190,7 +207,8 @@ class EnergyRecorderTest(unittest.TestCase):
         delete_mock.assert_called_once_with(
             EnergyRecorder.energy_recorder_api["uri"],
             auth=EnergyRecorder.energy_recorder_api["auth"],
-            headers=self.request_headers
+            headers=self.request_headers,
+            timeout=EnergyRecorder.CONNECTION_TIMOUT
         )
 
     @energy.enable_recording
@@ -206,13 +224,7 @@ class EnergyRecorderTest(unittest.TestCase):
     @mock.patch("functest.energy.energy.EnergyRecorder.get_current_scenario",
                 return_value=None)
     @mock.patch("functest.energy.energy.EnergyRecorder")
-    @mock.patch("functest.utils.functest_utils.get_pod_name",
-                return_value="MOCK_POD")
-    @mock.patch("functest.utils.functest_utils.get_functest_config",
-                side_effect=config_loader_mock)
     def test_decorators(self,
-                        loader_mock=None,
-                        pod_mock=None,
                         recorder_mock=None,
                         cur_scenario_mock=None):
         """Test energy module decorators."""
@@ -264,10 +276,14 @@ class EnergyRecorderTest(unittest.TestCase):
                 side_effect=config_loader_mock)
     @mock.patch("functest.utils.functest_utils.get_pod_name",
                 return_value="MOCK_POD")
-    def test_load_config(self, loader_mock=None, pod_mock=None):
+    @mock.patch("functest.energy.energy.requests.get",
+                return_value=API_OK)
+    def test_load_config(self, loader_mock=None, pod_mock=None,
+                         get_mock=None):
         """Test load config."""
         EnergyRecorder.energy_recorder_api = None
         EnergyRecorder.load_config()
+
         self.assertEquals(
             EnergyRecorder.energy_recorder_api["auth"],
             ("user", "password")
@@ -281,7 +297,10 @@ class EnergyRecorderTest(unittest.TestCase):
                 side_effect=config_loader_mock_no_creds)
     @mock.patch("functest.utils.functest_utils.get_pod_name",
                 return_value="MOCK_POD")
-    def test_load_config_no_creds(self, loader_mock=None, pod_mock=None):
+    @mock.patch("functest.energy.energy.requests.get",
+                return_value=API_OK)
+    def test_load_config_no_creds(self, loader_mock=None, pod_mock=None,
+                                  get_mock=None):
         """Test load config without creds."""
         EnergyRecorder.energy_recorder_api = None
         EnergyRecorder.load_config()
@@ -295,7 +314,10 @@ class EnergyRecorderTest(unittest.TestCase):
                 return_value=None)
     @mock.patch("functest.utils.functest_utils.get_pod_name",
                 return_value="MOCK_POD")
-    def test_load_config_ex(self, loader_mock=None, pod_mock=None):
+    @mock.patch("functest.energy.energy.requests.get",
+                return_value=API_OK)
+    def test_load_config_ex(self, loader_mock=None, pod_mock=None,
+                            get_mock=None):
         """Test load config with exception."""
         with self.assertRaises(AssertionError):
             EnergyRecorder.energy_recorder_api = None
@@ -303,6 +325,20 @@ class EnergyRecorderTest(unittest.TestCase):
         self.assertEquals(EnergyRecorder.energy_recorder_api, None)
 
     @mock.patch("functest.utils.functest_utils.get_functest_config",
+                side_effect=config_loader_mock)
+    @mock.patch("functest.utils.functest_utils.get_pod_name",
+                return_value="MOCK_POD")
+    @mock.patch("functest.energy.energy.requests.get",
+                return_value=API_KO)
+    def test_load_config_api_ko(self, loader_mock=None, pod_mock=None,
+                                get_mock=None):
+        """Test load config with API unavailable."""
+        EnergyRecorder.energy_recorder_api = None
+        EnergyRecorder.load_config()
+        self.assertEquals(EnergyRecorder.energy_recorder_api["available"],
+                          False)
+
+    @mock.patch("functest.utils.functest_utils.get_functest_config",
                 return_value=None)
     @mock.patch("functest.utils.functest_utils.get_pod_name",
                 return_value="MOCK_POD")