Return Functest status to Jenkins

It avoids mixing Functest and Features CI issues by simply exiting the
status of Functest testcases. The result of third-party testcases are
printed into console but are no longer taken into account into the
global status.

Change-Id: I040ff54780db2ddb6d81993839fee09f5e472323
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

2 files changed, 21 insertions, 11 deletions

diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py
index a129ea73..d4acd9c5 100644
--- a/functest/ci/run_tests.py
+++ b/functest/ci/run_tests.py
@@ -124,16 +124,18 @@ class Runner(object):
                 self.executed_test_cases[test.get_name()] = test_case
                 if self.clean_flag:
                     if test_case.create_snapshot() != test_case.EX_OK:
-                        return result
+                        return testcase.TestCase.EX_RUN_ERROR
                 try:
                     kwargs = run_dict['args']
-                    result = test_case.run(**kwargs)
+                    test_case.run(**kwargs)
                 except KeyError:
-                    result = test_case.run()
-                if result == testcase.TestCase.EX_OK:
-                    if self.report_flag:
-                        test_case.push_to_db()
+                    test_case.run()
+                if self.report_flag:
+                    test_case.push_to_db()
+                if test.get_project() == "functest":
                     result = test_case.is_successful()
+                else:
+                    result = testcase.TestCase.EX_OK
                 logger.info("Test result:\n\n%s\n", test_case)
                 if self.clean_flag:
                     test_case.clean()
@@ -157,10 +159,12 @@ class Runner(object):
         else:
             logger.info("Running tier '%s'" % tier_name)
             for test in tests:
-                result = self.run_test(test)
-                if result != testcase.TestCase.EX_OK:
+                self.run_test(test)
+                test_case = self.executed_test_cases[test.get_name()]
+                if test_case.is_successful() != testcase.TestCase.EX_OK:
                     logger.error("The test case '%s' failed.", test.get_name())
-                    self.overall_result = Result.EX_ERROR
+                    if test.get_project() == "functest":
+                        self.overall_result = Result.EX_ERROR
                     if test.is_blocking():
                         raise BlockingTestFailed(
                             "The test case {} failed and is blocking".format(
diff --git a/functest/tests/unit/ci/test_run_tests.py b/functest/tests/unit/ci/test_run_tests.py
index 7495c40e..bc95f8f3 100644
--- a/functest/tests/unit/ci/test_run_tests.py
+++ b/functest/tests/unit/ci/test_run_tests.py
@@ -31,6 +31,10 @@ class RunTestsTesting(unittest.TestCase):
 
     def setUp(self):
         self.runner = run_tests.Runner()
+        mock_test_case = mock.Mock()
+        mock_test_case.is_successful.return_value = TestCase.EX_OK
+        self.runner.executed_test_cases['test1'] = mock_test_case
+        self.runner.executed_test_cases['test2'] = mock_test_case
         self.sep = 'test_sep'
         self.creds = {'OS_AUTH_URL': 'http://test_ip:test_port/v2.0',
                       'OS_USERNAME': 'test_os_username',
@@ -191,8 +195,10 @@ class RunTestsTesting(unittest.TestCase):
     @mock.patch('functest.ci.run_tests.Runner.summary')
     def test_main_tier(self, *mock_methods):
         mock_tier = mock.Mock()
+        test_mock = mock.Mock()
+        test_mock.get_name.return_value = 'test1'
         args = {'get_name.return_value': 'tier_name',
-                'get_tests.return_value': ['test_name']}
+                'get_tests.return_value': [test_mock]}
         mock_tier.configure_mock(**args)
         kwargs = {'test': 'tier_name', 'noclean': True, 'report': True}
         args = {'get_tier.return_value': mock_tier,
@@ -201,7 +207,7 @@ class RunTestsTesting(unittest.TestCase):
         self.runner._tiers.configure_mock(**args)
         self.assertEqual(self.runner.main(**kwargs),
                          run_tests.Result.EX_OK)
-        mock_methods[1].assert_called_once_with('test_name')
+        mock_methods[1].assert_called()
 
     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
     @mock.patch('functest.ci.run_tests.Runner.run_test',