summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorCédric Ollivier <cedric.ollivier@orange.com>2016-12-15 16:54:45 +0100
committerCédric Ollivier <cedric.ollivier@orange.com>2016-12-15 18:24:38 +0100
commit0d875d44c3d67e049c165365add1084467143b62 (patch)
treeeb5cb1bab09f90e9196371c2aa446a11ef085610
parent41ccc6e8a54f11e0014707730bf3ef80385a01fa (diff)
Add check_criteria() in testcase_base
It allows checking testcase results and then returning an error value to CI. Now run() should return EX_OK even if criteria is FAILED. It also fixes an issue induced by [1]. [1] https://gerrit.opnfv.org/gerrit/#/c/24561/ Change-Id: Iaa095696f9dcb87f8cc95179b4f404b092d1e23e Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
-rwxr-xr-x[-rw-r--r--]functest/ci/run_tests.py7
-rw-r--r--functest/core/testcase_base.py10
-rwxr-xr-xfunctest/opnfv_tests/sdn/odl/odl.py6
-rw-r--r--functest/tests/unit/core/test_testcase_base.py17
-rw-r--r--[-rwxr-xr-x]functest/tests/unit/odl/test_odl.py12
5 files changed, 36 insertions, 16 deletions
diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py
index d6991f66b..557ba08dd 100644..100755
--- a/functest/ci/run_tests.py
+++ b/functest/ci/run_tests.py
@@ -141,9 +141,10 @@ def run_test(test, tier_name):
cls = getattr(module, run_dict['class'])
test_case = cls()
result = test_case.run()
- if (result == testcase_base.TestcaseBase.EX_OK and
- GlobalVariables.REPORT_FLAG):
- test_case.push_to_db()
+ if result == testcase_base.TestcaseBase.EX_OK:
+ if GlobalVariables.REPORT_FLAG:
+ test_case.push_to_db()
+ result = test_case.check_criteria()
except ImportError:
logger.exception("Cannot import module {}".format(
run_dict['module']))
diff --git a/functest/core/testcase_base.py b/functest/core/testcase_base.py
index e869803d8..838b63983 100644
--- a/functest/core/testcase_base.py
+++ b/functest/core/testcase_base.py
@@ -18,6 +18,7 @@ class TestcaseBase(object):
EX_OK = os.EX_OK
EX_RUN_ERROR = os.EX_SOFTWARE
EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1
+ EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2
logger = ft_logger.Logger(__name__).getLogger()
@@ -29,6 +30,15 @@ class TestcaseBase(object):
self.start_time = ""
self.stop_time = ""
+ def check_criteria(self):
+ try:
+ assert self.criteria
+ if self.criteria == 'PASS':
+ return TestcaseBase.EX_OK
+ except:
+ self.logger.error("Please run test before checking the results")
+ return TestcaseBase.EX_TESTCASE_FAILED
+
def run(self, **kwargs):
self.logger.error("Run must be implemented")
return TestcaseBase.EX_RUN_ERROR
diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py
index bc9f1bb7f..7f44ae69c 100755
--- a/functest/opnfv_tests/sdn/odl/odl.py
+++ b/functest/opnfv_tests/sdn/odl/odl.py
@@ -60,6 +60,7 @@ class ODLTests(testcase_base.TestcaseBase):
logger = ft_logger.Logger("opendaylight").getLogger()
def __init__(self):
+ testcase_base.TestcaseBase.__init__(self)
self.case_name = "odl"
@classmethod
@@ -89,7 +90,6 @@ class ODLTests(testcase_base.TestcaseBase):
self.details = {}
self.details['description'] = result.suite.name
self.details['tests'] = visitor.get_data()
- return self.criteria
def main(self, **kwargs):
dirs = [self.basic_suite_dir, self.neutron_suite_dir]
@@ -128,10 +128,8 @@ class ODLTests(testcase_base.TestcaseBase):
self.logger.info("\n" + stdout.read())
self.logger.info("ODL results were successfully generated")
try:
- test_res = self.parse_results()
+ self.parse_results()
self.logger.info("ODL results were successfully parsed")
- if test_res is not "PASS":
- return self.EX_RUN_ERROR
except RobotError as e:
self.logger.error("Run tests before publishing: %s" %
e.message)
diff --git a/functest/tests/unit/core/test_testcase_base.py b/functest/tests/unit/core/test_testcase_base.py
index fe7b0d054..181eed94b 100644
--- a/functest/tests/unit/core/test_testcase_base.py
+++ b/functest/tests/unit/core/test_testcase_base.py
@@ -24,7 +24,7 @@ class TestcaseBaseTesting(unittest.TestCase):
self.test.case_name = "base"
self.test.start_time = "1"
self.test.stop_time = "2"
- self.test.criteria = "100"
+ self.test.criteria = "PASS"
self.test.details = {"Hello": "World"}
def test_run_unimplemented(self):
@@ -82,6 +82,21 @@ class TestcaseBaseTesting(unittest.TestCase):
self.test.project, self.test.case_name, self.test.start_time,
self.test.stop_time, self.test.criteria, self.test.details)
+ def test_check_criteria_missing(self):
+ self.test.criteria = None
+ self.assertEqual(self.test.check_criteria(),
+ testcase_base.TestcaseBase.EX_TESTCASE_FAILED)
+
+ def test_check_criteria_failed(self):
+ self.test.criteria = 'FAILED'
+ self.assertEqual(self.test.check_criteria(),
+ testcase_base.TestcaseBase.EX_TESTCASE_FAILED)
+
+ def test_check_criteria_pass(self):
+ self.test.criteria = 'PASS'
+ self.assertEqual(self.test.check_criteria(),
+ testcase_base.TestcaseBase.EX_OK)
+
if __name__ == "__main__":
unittest.main(verbosity=2)
diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py
index c29bfd7f3..9a63ffb01 100755..100644
--- a/functest/tests/unit/odl/test_odl.py
+++ b/functest/tests/unit/odl/test_odl.py
@@ -192,8 +192,7 @@ class ODLTesting(unittest.TestCase):
def test_main(self, *args):
with mock.patch.object(self.test, 'set_robotframework_vars',
return_value=True), \
- mock.patch.object(self.test, 'parse_results',
- return_value="PASS"):
+ mock.patch.object(self.test, 'parse_results'):
self._test_main(testcase_base.TestcaseBase.EX_OK, *args)
@mock.patch('os.remove')
@@ -202,8 +201,7 @@ class ODLTesting(unittest.TestCase):
def test_main_makedirs_oserror17(self, *args):
with mock.patch.object(self.test, 'set_robotframework_vars',
return_value=True), \
- mock.patch.object(self.test, 'parse_results',
- return_value="PASS"):
+ mock.patch.object(self.test, 'parse_results'):
self._test_main(testcase_base.TestcaseBase.EX_OK, *args)
@mock.patch('os.remove')
@@ -212,8 +210,7 @@ class ODLTesting(unittest.TestCase):
def test_main_testcases_in_failure(self, *args):
with mock.patch.object(self.test, 'set_robotframework_vars',
return_value=True), \
- mock.patch.object(self.test, 'parse_results',
- return_value="PASS"):
+ mock.patch.object(self.test, 'parse_results'):
self._test_main(testcase_base.TestcaseBase.EX_OK, *args)
@mock.patch('os.remove', side_effect=OSError)
@@ -222,8 +219,7 @@ class ODLTesting(unittest.TestCase):
def test_main_remove_oserror(self, *args):
with mock.patch.object(self.test, 'set_robotframework_vars',
return_value=True), \
- mock.patch.object(self.test, 'parse_results',
- return_value="PASS"):
+ mock.patch.object(self.test, 'parse_results'):
self._test_main(testcase_base.TestcaseBase.EX_OK, *args)
def _test_run_missing_env_var(self, var):