diff options
| author |   Cédric Ollivier <cedric.ollivier@orange.com> | 2017-04-24 08:58:20 +0200 |
|---|---|---|
| committer | Cédric Ollivier <cedric.ollivier@orange.com> | 2017-04-26 09:35:47 +0200 |
| commit | 37c98a6fa101be2b32129dd5d71ad750f5245b88 (patch) | |
| tree | 9b3f8700cb5af4054593c667a7f82aa160e32a8f /functest/core | |
| parent | e620488a6747318c40eb973c2607ae6d44e95b8f (diff) | |
Manage criteria in TestCase
It converts all criteria values to the corresponding percent in
functest/ci/testcases.yaml.
Result is expected to be equal or greater than criteria. If both are 0,
result is considered as false.
It is compatible with the old behavior but warns to update.
It will allow a safer remove.
It also fixes a bug in test_tempest to allow merging [1] and
tier_handler.py which required that type criteria was str.
[1] https://gerrit.opnfv.org/gerrit/#/c/27949/
Change-Id: Ib6edcfa3103b7d51b0bdc83119f1cea2a8be9fbc
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
Diffstat (limited to 'functest/core')
| -rw-r--r-- | functest/core/feature.py | 4 | ||||
| -rw-r--r-- | functest/core/testcase.py | 22 |
2 files changed, 19 insertions, 7 deletions
diff --git a/functest/core/feature.py b/functest/core/feature.py index 08500a263..d65f5a3c1 100644 --- a/functest/core/feature.py +++ b/functest/core/feature.py @@ -74,11 +74,11 @@ class Feature(base.TestCase): """ self.start_time = time.time() exit_code = base.TestCase.EX_RUN_ERROR - self.result = "FAIL" + self.result = 0 try: if self.execute(**kwargs) == 0: exit_code = base.TestCase.EX_OK - self.result = 'PASS' + self.result = 100 ft_utils.logger_test_results( self.project_name, self.case_name, self.result, self.details) diff --git a/functest/core/testcase.py b/functest/core/testcase.py index b9dcbb2db..3f191b407 100644 --- a/functest/core/testcase.py +++ b/functest/core/testcase.py @@ -38,6 +38,7 @@ class TestCase(object): self.details = {} self.project_name = kwargs.get('project_name', 'functest') self.case_name = kwargs.get('case_name', '') + self.criteria = kwargs.get('criteria', 100) self.result = "" self.start_time = "" self.stop_time = "" @@ -55,9 +56,19 @@ class TestCase(object): TestCase.EX_TESTCASE_FAILED otherwise. """ try: - assert self.result - if self.result == 'PASS': - return TestCase.EX_OK + assert self.criteria + if isinstance(self.result, int) and isinstance(self.criteria, int): + if self.result >= self.criteria: + return TestCase.EX_OK + else: + # Backward compatibility + # It must be removed as soon as TestCase subclasses + # stop setting result = 'PASS' or 'FAIL'. + # In this case criteria is unread. + self.logger.warning( + "Please update result which must be an int!") + if self.result == 'PASS': + return TestCase.EX_OK except AssertionError: self.logger.error("Please run test before checking the results") return TestCase.EX_TESTCASE_FAILED @@ -110,12 +121,13 @@ class TestCase(object): try: assert self.project_name assert self.case_name - assert self.result assert self.start_time assert self.stop_time + pub_result = 'PASS' if self.check_result( + ) == TestCase.EX_OK else 'FAIL' if ft_utils.push_results_to_db( self.project_name, self.case_name, self.start_time, - self.stop_time, self.result, self.details): + self.stop_time, pub_result, self.details): self.logger.info("The results were successfully pushed to DB") return TestCase.EX_OK else: |