aboutsummaryrefslogtreecommitdiffstats
path: root/functest/tests
diff options
context:
space:
mode:
authorCédric Ollivier <cedric.ollivier@orange.com>2017-04-24 08:58:20 +0200
committerCédric Ollivier <cedric.ollivier@orange.com>2017-04-26 09:35:47 +0200
commit37c98a6fa101be2b32129dd5d71ad750f5245b88 (patch)
tree9b3f8700cb5af4054593c667a7f82aa160e32a8f /functest/tests
parente620488a6747318c40eb973c2607ae6d44e95b8f (diff)
Manage criteria in TestCase
It converts all criteria values to the corresponding percent in functest/ci/testcases.yaml. Result is expected to be equal or greater than criteria. If both are 0, result is considered as false. It is compatible with the old behavior but warns to update. It will allow a safer remove. It also fixes a bug in test_tempest to allow merging [1] and tier_handler.py which required that type criteria was str. [1] https://gerrit.opnfv.org/gerrit/#/c/27949/ Change-Id: Ib6edcfa3103b7d51b0bdc83119f1cea2a8be9fbc Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
Diffstat (limited to 'functest/tests')
-rw-r--r--functest/tests/unit/core/test_feature.py4
-rw-r--r--functest/tests/unit/core/test_testcase.py69
-rw-r--r--functest/tests/unit/openstack/tempest/test_tempest.py2
-rw-r--r--functest/tests/unit/utils/test_functest_utils.py9
4 files changed, 67 insertions, 17 deletions
diff --git a/functest/tests/unit/core/test_feature.py b/functest/tests/unit/core/test_feature.py
index 97075223..993da5a0 100644
--- a/functest/tests/unit/core/test_feature.py
+++ b/functest/tests/unit/core/test_feature.py
@@ -35,9 +35,9 @@ class FeatureTestingBase(unittest.TestCase):
def _test_run(self, status, mock_method=None):
self.assertEqual(self.feature.run(cmd=self._cmd), status)
if status == testcase.TestCase.EX_OK:
- self.assertEqual(self.feature.result, 'PASS')
+ self.assertEqual(self.feature.result, 100)
else:
- self.assertEqual(self.feature.result, 'FAIL')
+ self.assertEqual(self.feature.result, 0)
mock_method.assert_has_calls([mock.call(), mock.call()])
self.assertEqual(self.feature.start_time, 1)
self.assertEqual(self.feature.stop_time, 2)
diff --git a/functest/tests/unit/core/test_testcase.py b/functest/tests/unit/core/test_testcase.py
index b93f50d6..cc8446d8 100644
--- a/functest/tests/unit/core/test_testcase.py
+++ b/functest/tests/unit/core/test_testcase.py
@@ -28,13 +28,14 @@ class TestCaseTesting(unittest.TestCase):
_case_name = "base"
_project_name = "functest"
+ _published_result = "PASS"
def setUp(self):
self.test = testcase.TestCase(case_name=self._case_name,
project_name=self._project_name)
self.test.start_time = "1"
self.test.stop_time = "2"
- self.test.result = "PASS"
+ self.test.result = 100
self.test.details = {"Hello": "World"}
def test_run_unimplemented(self):
@@ -56,10 +57,6 @@ class TestCaseTesting(unittest.TestCase):
self.test.case_name = None
self._test_missing_attribute()
- def test_missing_criteria(self):
- self.test.result = None
- self._test_missing_attribute()
-
def test_missing_start_time(self):
self.test.start_time = None
self._test_missing_attribute()
@@ -76,7 +73,7 @@ class TestCaseTesting(unittest.TestCase):
testcase.TestCase.EX_OK)
mock_function.assert_called_once_with(
self._project_name, self._case_name, self.test.start_time,
- self.test.stop_time, self.test.result, self.test.details)
+ self.test.stop_time, self._published_result, self.test.details)
@mock.patch('functest.utils.functest_utils.push_results_to_db',
return_value=False)
@@ -85,7 +82,7 @@ class TestCaseTesting(unittest.TestCase):
testcase.TestCase.EX_PUSH_TO_DB_ERROR)
mock_function.assert_called_once_with(
self._project_name, self._case_name, self.test.start_time,
- self.test.stop_time, self.test.result, self.test.details)
+ self.test.stop_time, self._published_result, self.test.details)
@mock.patch('functest.utils.functest_utils.push_results_to_db',
return_value=True)
@@ -94,7 +91,33 @@ class TestCaseTesting(unittest.TestCase):
testcase.TestCase.EX_OK)
mock_function.assert_called_once_with(
self._project_name, self._case_name, self.test.start_time,
- self.test.stop_time, self.test.result, self.test.details)
+ self.test.stop_time, self._published_result, self.test.details)
+
+ @mock.patch('functest.utils.functest_utils.push_results_to_db',
+ return_value=True)
+ def test_push_to_db_res_ko(self, mock_function=None):
+ self.test.result = 0
+ self.assertEqual(self.test.push_to_db(),
+ testcase.TestCase.EX_OK)
+ mock_function.assert_called_once_with(
+ self._project_name, self._case_name, self.test.start_time,
+ self.test.stop_time, 'FAIL', self.test.details)
+
+ @mock.patch('functest.utils.functest_utils.push_results_to_db',
+ return_value=True)
+ def test_push_to_db_both_ko(self, mock_function=None):
+ self.test.result = 0
+ self.test.criteria = 0
+ self.assertEqual(self.test.push_to_db(),
+ testcase.TestCase.EX_OK)
+ mock_function.assert_called_once_with(
+ self._project_name, self._case_name, self.test.start_time,
+ self.test.stop_time, 'FAIL', self.test.details)
+
+ def test_check_criteria_missing(self):
+ self.test.criteria = None
+ self.assertEqual(self.test.check_result(),
+ testcase.TestCase.EX_TESTCASE_FAILED)
def test_check_result_missing(self):
self.test.result = None
@@ -102,15 +125,43 @@ class TestCaseTesting(unittest.TestCase):
testcase.TestCase.EX_TESTCASE_FAILED)
def test_check_result_failed(self):
- self.test.result = 'FAILED'
+ # Backward compatibility
+ # It must be removed as soon as TestCase subclasses
+ # stop setting result = 'PASS' or 'FAIL'.
+ self.test.result = 'FAIL'
self.assertEqual(self.test.check_result(),
testcase.TestCase.EX_TESTCASE_FAILED)
def test_check_result_pass(self):
+ # Backward compatibility
+ # It must be removed as soon as TestCase subclasses
+ # stop setting result = 'PASS' or 'FAIL'.
self.test.result = 'PASS'
self.assertEqual(self.test.check_result(),
testcase.TestCase.EX_OK)
+ def test_check_result_lt(self):
+ self.test.result = 50
+ self.assertEqual(self.test.check_result(),
+ testcase.TestCase.EX_TESTCASE_FAILED)
+
+ def test_check_result_eq(self):
+ self.test.result = 100
+ self.assertEqual(self.test.check_result(),
+ testcase.TestCase.EX_OK)
+
+ def test_check_result_gt(self):
+ self.test.criteria = 50
+ self.test.result = 100
+ self.assertEqual(self.test.check_result(),
+ testcase.TestCase.EX_OK)
+
+ def test_check_result_zero(self):
+ self.test.criteria = 0
+ self.test.result = 0
+ self.assertEqual(self.test.check_result(),
+ testcase.TestCase.EX_TESTCASE_FAILED)
+
if __name__ == "__main__":
unittest.main(verbosity=2)
diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py
index 34031b40..e1653a40 100644
--- a/functest/tests/unit/openstack/tempest/test_tempest.py
+++ b/functest/tests/unit/openstack/tempest/test_tempest.py
@@ -105,7 +105,7 @@ class OSTempestTesting(unittest.TestCase):
self._test_generate_test_list_mode_default('full')
def test_parse_verifier_result_missing_verification_uuid(self):
- self.tempestcommon.VERIFICATION_ID = ''
+ self.tempestcommon.VERIFICATION_ID = None
with self.assertRaises(Exception):
self.tempestcommon.parse_verifier_result()
diff --git a/functest/tests/unit/utils/test_functest_utils.py b/functest/tests/unit/utils/test_functest_utils.py
index 7ab8b455..573fcb70 100644
--- a/functest/tests/unit/utils/test_functest_utils.py
+++ b/functest/tests/unit/utils/test_functest_utils.py
@@ -41,9 +41,8 @@ class FunctestUtilsTesting(unittest.TestCase):
self.status = 'test_status'
self.details = 'test_details'
self.db_url = 'test_db_url'
- self.success_rate = 2.0
- self.criteria = 'test_criteria==2.0'
- self.result = 'PASS'
+ self.criteria = 50
+ self.result = 75
self.start_date = 1482624000
self.stop_date = 1482624000
self.start_time = time.time()
@@ -567,7 +566,7 @@ class FunctestUtilsTesting(unittest.TestCase):
as mock_criteria:
mock_criteria.return_value = self.criteria
resp = functest_utils.check_success_rate(self.case_name,
- self.success_rate)
+ self.result)
self.assertEqual(resp, 'PASS')
def test_check_success_rate_failed(self):
@@ -575,7 +574,7 @@ class FunctestUtilsTesting(unittest.TestCase):
as mock_criteria:
mock_criteria.return_value = self.criteria
resp = functest_utils.check_success_rate(self.case_name,
- 3.0)
+ 0)
self.assertEqual(resp, 'FAIL')
# TODO: merge_dicts