Add check_criteria() in testcase_base

It allows checking testcase results and then returning an error value
to CI. Now run() should return EX_OK even if criteria is FAILED.
It also fixes an issue induced by [1].

[1] https://gerrit.opnfv.org/gerrit/#/c/24561/

Change-Id: Iaa095696f9dcb87f8cc95179b4f404b092d1e23e
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

5 files changed, 36 insertions, 16 deletions

diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py
index d6991f66b..557ba08dd 100644..100755
--- a/functest/ci/run_tests.py
+++ b/functest/ci/run_tests.py
@@ -141,9 +141,10 @@ def run_test(test, tier_name):
             cls = getattr(module, run_dict['class'])
             test_case = cls()
             result = test_case.run()
-            if (result == testcase_base.TestcaseBase.EX_OK and
-                    GlobalVariables.REPORT_FLAG):
-                test_case.push_to_db()
+            if result == testcase_base.TestcaseBase.EX_OK:
+                if GlobalVariables.REPORT_FLAG:
+                    test_case.push_to_db()
+                result = test_case.check_criteria()
         except ImportError:
             logger.exception("Cannot import module {}".format(
                 run_dict['module']))
diff --git a/functest/core/testcase_base.py b/functest/core/testcase_base.py
index e869803d8..838b63983 100644
--- a/functest/core/testcase_base.py
+++ b/functest/core/testcase_base.py
@@ -18,6 +18,7 @@ class TestcaseBase(object):
     EX_OK = os.EX_OK
     EX_RUN_ERROR = os.EX_SOFTWARE
     EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1
+    EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2
 
     logger = ft_logger.Logger(__name__).getLogger()
 
@@ -29,6 +30,15 @@ class TestcaseBase(object):
         self.start_time = ""
         self.stop_time = ""
 
+    def check_criteria(self):
+        try:
+            assert self.criteria
+            if self.criteria == 'PASS':
+                return TestcaseBase.EX_OK
+        except:
+            self.logger.error("Please run test before checking the results")
+        return TestcaseBase.EX_TESTCASE_FAILED
+
     def run(self, **kwargs):
         self.logger.error("Run must be implemented")
         return TestcaseBase.EX_RUN_ERROR
diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py
index bc9f1bb7f..7f44ae69c 100755
--- a/functest/opnfv_tests/sdn/odl/odl.py
+++ b/functest/opnfv_tests/sdn/odl/odl.py
@@ -60,6 +60,7 @@ class ODLTests(testcase_base.TestcaseBase):
     logger = ft_logger.Logger("opendaylight").getLogger()
 
     def __init__(self):
+        testcase_base.TestcaseBase.__init__(self)
         self.case_name = "odl"
 
     @classmethod
@@ -89,7 +90,6 @@ class ODLTests(testcase_base.TestcaseBase):
         self.details = {}
         self.details['description'] = result.suite.name
         self.details['tests'] = visitor.get_data()
-        return self.criteria
 
     def main(self, **kwargs):
         dirs = [self.basic_suite_dir, self.neutron_suite_dir]
@@ -128,10 +128,8 @@ class ODLTests(testcase_base.TestcaseBase):
                 self.logger.info("\n" + stdout.read())
             self.logger.info("ODL results were successfully generated")
             try:
-                test_res = self.parse_results()
+                self.parse_results()
                 self.logger.info("ODL results were successfully parsed")
-                if test_res is not "PASS":
-                    return self.EX_RUN_ERROR
             except RobotError as e:
                 self.logger.error("Run tests before publishing: %s" %
                                   e.message)
diff --git a/functest/tests/unit/core/test_testcase_base.py b/functest/tests/unit/core/test_testcase_base.py
index fe7b0d054..181eed94b 100644
--- a/functest/tests/unit/core/test_testcase_base.py
+++ b/functest/tests/unit/core/test_testcase_base.py
@@ -24,7 +24,7 @@ class TestcaseBaseTesting(unittest.TestCase):
         self.test.case_name = "base"
         self.test.start_time = "1"
         self.test.stop_time = "2"
-        self.test.criteria = "100"
+        self.test.criteria = "PASS"
         self.test.details = {"Hello": "World"}
 
     def test_run_unimplemented(self):
@@ -82,6 +82,21 @@ class TestcaseBaseTesting(unittest.TestCase):
             self.test.project, self.test.case_name, self.test.start_time,
             self.test.stop_time, self.test.criteria, self.test.details)
 
+    def test_check_criteria_missing(self):
+        self.test.criteria = None
+        self.assertEqual(self.test.check_criteria(),
+                         testcase_base.TestcaseBase.EX_TESTCASE_FAILED)
+
+    def test_check_criteria_failed(self):
+        self.test.criteria = 'FAILED'
+        self.assertEqual(self.test.check_criteria(),
+                         testcase_base.TestcaseBase.EX_TESTCASE_FAILED)
+
+    def test_check_criteria_pass(self):
+        self.test.criteria = 'PASS'
+        self.assertEqual(self.test.check_criteria(),
+                         testcase_base.TestcaseBase.EX_OK)
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)
diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py
index c29bfd7f3..9a63ffb01 100755..100644
--- a/functest/tests/unit/odl/test_odl.py
+++ b/functest/tests/unit/odl/test_odl.py
@@ -192,8 +192,7 @@ class ODLTesting(unittest.TestCase):
     def test_main(self, *args):
         with mock.patch.object(self.test, 'set_robotframework_vars',
                                return_value=True), \
-                mock.patch.object(self.test, 'parse_results',
-                                  return_value="PASS"):
+                mock.patch.object(self.test, 'parse_results'):
             self._test_main(testcase_base.TestcaseBase.EX_OK, *args)
 
     @mock.patch('os.remove')
@@ -202,8 +201,7 @@ class ODLTesting(unittest.TestCase):
     def test_main_makedirs_oserror17(self, *args):
         with mock.patch.object(self.test, 'set_robotframework_vars',
                                return_value=True), \
-                mock.patch.object(self.test, 'parse_results',
-                                  return_value="PASS"):
+                mock.patch.object(self.test, 'parse_results'):
             self._test_main(testcase_base.TestcaseBase.EX_OK, *args)
 
     @mock.patch('os.remove')
@@ -212,8 +210,7 @@ class ODLTesting(unittest.TestCase):
     def test_main_testcases_in_failure(self, *args):
         with mock.patch.object(self.test, 'set_robotframework_vars',
                                return_value=True), \
-                mock.patch.object(self.test, 'parse_results',
-                                  return_value="PASS"):
+                mock.patch.object(self.test, 'parse_results'):
             self._test_main(testcase_base.TestcaseBase.EX_OK, *args)
 
     @mock.patch('os.remove', side_effect=OSError)
@@ -222,8 +219,7 @@ class ODLTesting(unittest.TestCase):
     def test_main_remove_oserror(self, *args):
         with mock.patch.object(self.test, 'set_robotframework_vars',
                                return_value=True), \
-                mock.patch.object(self.test, 'parse_results',
-                                  return_value="PASS"):
+                mock.patch.object(self.test, 'parse_results'):
             self._test_main(testcase_base.TestcaseBase.EX_OK, *args)
 
     def _test_run_missing_env_var(self, var):