Get HA test case results on failure

Now, yardstick HA test cases (which have 'sla action: monitor') do
not output results if the test case SLA validation fails.
This patch modifies the task runner and the serviceHA scenario so
that during an SLA validation failure the test result data is
captured correctly.

Do similar changes in the general scenario.

JIRA: YARDSTICK-1115

Change-Id: I648a8b229600c0ad089320ac3f803698f73aa800
Signed-off-by: Miikka Koistinen <miikka.koistinen@nokia.com>

1 files changed, 18 insertions, 11 deletions

diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py
index 955b8cae2..697cc007f 100644
--- a/yardstick/benchmark/core/task.py
+++ b/yardstick/benchmark/core/task.py
@@ -112,9 +112,9 @@ class Task(object):     # pragma: no cover
                 continue
 
             try:
-                data = self._run(tasks[i]['scenarios'],
-                                 tasks[i]['run_in_parallel'],
-                                 output_config)
+                success, data = self._run(tasks[i]['scenarios'],
+                                          tasks[i]['run_in_parallel'],
+                                          output_config)
             except KeyboardInterrupt:
                 raise
             except Exception:  # pylint: disable=broad-except
@@ -123,9 +123,15 @@ class Task(object):     # pragma: no cover
                 testcases[tasks[i]['case_name']] = {'criteria': 'FAIL',
                                                     'tc_data': []}
             else:
-                LOG.info('Testcase: "%s" SUCCESS!!!', tasks[i]['case_name'])
-                testcases[tasks[i]['case_name']] = {'criteria': 'PASS',
-                                                    'tc_data': data}
+                if success:
+                    LOG.info('Testcase: "%s" SUCCESS!!!', tasks[i]['case_name'])
+                    testcases[tasks[i]['case_name']] = {'criteria': 'PASS',
+                                                        'tc_data': data}
+                else:
+                    LOG.error('Testcase: "%s" FAILED!!!', tasks[i]['case_name'],
+                              exc_info=True)
+                    testcases[tasks[i]['case_name']] = {'criteria': 'FAIL',
+                                                        'tc_data': data}
 
             if args.keep_deploy:
                 # keep deployment, forget about stack
@@ -240,6 +246,7 @@ class Task(object):     # pragma: no cover
 
         background_runners = []
 
+        task_success = True
         result = []
         # Start all background scenarios
         for scenario in filter(_is_background_scenario, scenarios):
@@ -258,8 +265,8 @@ class Task(object):     # pragma: no cover
             for runner in runners:
                 status = runner_join(runner, background_runners, self.outputs, result)
                 if status != 0:
-                    raise RuntimeError(
-                        "{0} runner status {1}".format(runner.__execution_type__, status))
+                    LOG.error("%s runner status %s", runner.__execution_type__, status)
+                    task_success = False
                 LOG.info("Runner ended")
         else:
             # run serially
@@ -271,8 +278,8 @@ class Task(object):     # pragma: no cover
                         LOG.error('Scenario NO.%s: "%s" ERROR!',
                                   scenarios.index(scenario) + 1,
                                   scenario.get('type'))
-                        raise RuntimeError(
-                            "{0} runner status {1}".format(runner.__execution_type__, status))
+                        LOG.error("%s runner status %s", runner.__execution_type__, status)
+                        task_success = False
                     LOG.info("Runner ended")
 
         # Abort background runners
@@ -289,7 +296,7 @@ class Task(object):     # pragma: no cover
             base_runner.Runner.release(runner)
 
             print("Background task ended")
-        return result
+        return task_success, result
 
     def atexit_handler(self):
         """handler for process termination"""