diff options
| -rwxr-xr-x | functest/ci/run_tests.py | 15 | ||||
| -rw-r--r-- | functest/core/pytest_suite_runner.py | 42 | ||||
| -rwxr-xr-x | functest/opnfv_tests/openstack/refstack_client/refstack_client.py | 24 | ||||
| -rwxr-xr-x | functest/opnfv_tests/openstack/refstack_client/tempest_conf.py | 8 | ||||
| -rwxr-xr-x | functest/opnfv_tests/sdn/odl/odl.py | 2 | ||||
| -rw-r--r-- | functest/tests/unit/core/test_pytest_suite_runner.py | 76 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/refstack_client/test_refstack_client.py | 24 |
7 files changed, 112 insertions, 79 deletions
diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py index f973b616..95353c87 100755 --- a/functest/ci/run_tests.py +++ b/functest/ci/run_tests.py @@ -122,10 +122,9 @@ class Runner(object): if not test.is_enabled(): raise TestNotEnabled( "The test case {} is not enabled".format(test.get_name())) - test_name = test.get_name() logger.info("\n") # blank line self.print_separator("=") - logger.info("Running test case '%s'..." % test_name) + logger.info("Running test case '%s'..." % test.get_name()) self.print_separator("=") logger.debug("\n%s" % test) self.source_rc_file() @@ -133,17 +132,17 @@ class Runner(object): if test.needs_clean() and self.clean_flag: self.generate_os_snapshot() - flags = (" -t %s" % (test_name)) + flags = " -t %s" % test.get_name() if self.report_flag: flags += " -r" result = testcase.TestCase.EX_RUN_ERROR - run_dict = self.get_run_dict(test_name) + run_dict = self.get_run_dict(test.get_name()) if run_dict: try: module = importlib.import_module(run_dict['module']) cls = getattr(module, run_dict['class']) - test_dict = ft_utils.get_dict_by_test(test_name) + test_dict = ft_utils.get_dict_by_test(test.get_name()) test_case = cls(**test_dict) self.executed_test_cases.append(test_case) try: @@ -168,7 +167,7 @@ class Runner(object): if test.needs_clean() and self.clean_flag: self.cleanup() if result != testcase.TestCase.EX_OK: - logger.error("The test case '%s' failed. " % test_name) + logger.error("The test case '%s' failed. " % test.get_name()) self.overall_result = Result.EX_ERROR if test.is_blocking(): raise BlockingTestFailed( @@ -242,8 +241,10 @@ class Runner(object): return Result.EX_ERROR else: self.run_all(_tiers) + except BlockingTestFailed: + pass except Exception: - logger.exception("Runner failed") + logger.exception("Failures when running testcase(s)") self.overall_result = Result.EX_ERROR msg = prettytable.PrettyTable( diff --git a/functest/core/pytest_suite_runner.py b/functest/core/pytest_suite_runner.py index a6e47660..efcef7b6 100644 --- a/functest/core/pytest_suite_runner.py +++ b/functest/core/pytest_suite_runner.py @@ -7,6 +7,8 @@ # pylint: disable=missing-docstring +from __future__ import division + import logging import time import unittest @@ -46,32 +48,14 @@ class PyTestSuiteRunner(testcase.TestCase): stream=stream, verbosity=2).run(self.suite) self.logger.debug("\n\n%s", stream.getvalue()) self.stop_time = time.time() - - if result.errors: - self.logger.error('Number of errors in test suite - ' + - str(len(result.errors))) - for test, message in result.errors: - self.logger.error(str(test) + " ERROR with " + message) - - if result.failures: - self.logger.error('Number of failures in test suite - ' + - str(len(result.failures))) - for test, message in result.failures: - self.logger.error(str(test) + " FAILED with " + message) - - # a result can be PASS or FAIL - # But in this case it means that the Execution was OK - # we shall distinguish Execution Error from FAIL results - # TestCase.EX_RUN_ERROR means that the test case was not run - # not that it was run but the result was FAIL - exit_code = testcase.TestCase.EX_OK - if ((result.errors and len(result.errors) > 0) or - (result.failures and len(result.failures) > 0)): - self.logger.info("%s FAILED", self.case_name) - self.result = 0 - else: - self.logger.info("%s OK", self.case_name) - self.result = 100 - - self.details = {} - return exit_code + self.details = {"failures": result.failures, + "errors": result.errors} + try: + self.result = 100 * ( + (result.testsRun - (len(result.failures) + + len(result.errors))) / + result.testsRun) + return testcase.TestCase.EX_OK + except ZeroDivisionError: + self.logger.error("No test has been run") + return testcase.TestCase.EX_RUN_ERROR diff --git a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py index 2a2718dd..5f1f3a1d 100755 --- a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py +++ b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py @@ -33,9 +33,9 @@ class RefstackClient(testcase.TestCase): if "case_name" not in kwargs: kwargs["case_name"] = "refstack_defcore" super(RefstackClient, self).__init__(**kwargs) - self.FUNCTEST_TEST = CONST.dir_functest_test - self.CONF_PATH = CONST.refstack_tempest_conf_path - self.DEFCORE_LIST = CONST.refstack_defcore_list + self.FUNCTEST_TEST = CONST.__getattribute__('dir_functest_test') + self.CONF_PATH = CONST.__getattribute__('refstack_tempest_conf_path') + self.DEFCORE_LIST = CONST.__getattribute__('refstack_defcore_list') self.confpath = os.path.join(self.FUNCTEST_TEST, self.CONF_PATH) self.defcorelist = os.path.join(self.FUNCTEST_TEST, @@ -45,7 +45,7 @@ class RefstackClient(testcase.TestCase): cmd = ("cd {0};" ". .venv/bin/activate;" - "cd -;".format(CONST.dir_refstack_client)) + "cd -;".format(CONST.__getattribute__('dir_refstack_client'))) ft_utils.execute_command(cmd) def run_defcore(self, conf, testlist): @@ -53,7 +53,7 @@ class RefstackClient(testcase.TestCase): cmd = ("cd {0};" "./refstack-client test -c {1} -v --test-list {2};" - "cd -;".format(CONST.dir_refstack_client, + "cd -;".format(CONST.__getattribute__('dir_refstack_client'), conf, testlist)) ft_utils.execute_command(cmd) @@ -63,16 +63,16 @@ class RefstackClient(testcase.TestCase): cmd = ("cd {0};" "./refstack-client test -c {1} -v --test-list {2};" - "cd -;".format(CONST.dir_refstack_client, + "cd -;".format(CONST.__getattribute__('dir_refstack_client'), self.confpath, self.defcorelist)) logger.info("Starting Refstack_defcore test case: '%s'." % cmd) header = ("Refstack environment:\n" " SUT: %s\n Scenario: %s\n Node: %s\n Date: %s\n" % - (CONST.INSTALLER_TYPE, - CONST.DEPLOY_SCENARIO, - CONST.NODE_NAME, + (CONST.__getattribute__('INSTALLER_TYPE'), + CONST.__getattribute__('DEPLOY_SCENARIO'), + CONST.__getattribute__('NODE_NAME'), time.strftime("%a %b %d %H:%M:%S %Z %Y"))) f_stdout = open( @@ -202,9 +202,9 @@ class RefstackClient(testcase.TestCase): class RefstackClientParser(object): def __init__(self): - self.FUNCTEST_TEST = CONST.dir_functest_test - self.CONF_PATH = CONST.refstack_tempest_conf_path - self.DEFCORE_LIST = CONST.refstack_defcore_list + self.FUNCTEST_TEST = CONST.__getattribute__('dir_functest_test') + self.CONF_PATH = CONST.__getattribute__('refstack_tempest_conf_path') + self.DEFCORE_LIST = CONST.__getattribute__('refstack_defcore_list') self.confpath = os.path.join(self.FUNCTEST_TEST, self.CONF_PATH) self.defcorelist = os.path.join(self.FUNCTEST_TEST, diff --git a/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py b/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py index 5c04253c..fbaad589 100755 --- a/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py +++ b/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py @@ -24,12 +24,14 @@ class TempestConf(object): self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id() self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir( self.VERIFIER_ID, self.DEPLOYMENT_ID) - self.confpath = os.path.join(CONST.dir_functest_test, - CONST.refstack_tempest_conf_path) + self.confpath = os.path.join( + CONST.__getattribute__('dir_functest_test'), + CONST.__getattribute__('refstack_tempest_conf_path')) def generate_tempestconf(self): try: - openstack_utils.source_credentials(CONST.openstack_creds) + openstack_utils.source_credentials( + CONST.__getattribute__('openstack_creds')) img_flavor_dict = conf_utils.create_tempest_resources( use_custom_images=True, use_custom_flavors=True) conf_utils.configure_tempest_defcore( diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py index 2f3dd74b..b2b0b77c 100755 --- a/functest/opnfv_tests/sdn/odl/odl.py +++ b/functest/opnfv_tests/sdn/odl/odl.py @@ -108,7 +108,7 @@ class ODLTests(testcase.TestCase): result.suite.statistics.critical.passed / result.suite.statistics.critical.total) except ZeroDivisionError: - self.__logger.error("No test has been ran") + self.__logger.error("No test has been run") self.start_time = timestamp_to_secs(result.suite.starttime) self.stop_time = timestamp_to_secs(result.suite.endtime) self.details = {} diff --git a/functest/tests/unit/core/test_pytest_suite_runner.py b/functest/tests/unit/core/test_pytest_suite_runner.py index 07ac7906..f317cdea 100644 --- a/functest/tests/unit/core/test_pytest_suite_runner.py +++ b/functest/tests/unit/core/test_pytest_suite_runner.py @@ -20,29 +20,71 @@ class PyTestSuiteRunnerTesting(unittest.TestCase): def setUp(self): self.psrunner = pytest_suite_runner.PyTestSuiteRunner() - self.result = mock.Mock() - attrs = {'errors': [('test1', 'error_msg1')], - 'failures': [('test2', 'failure_msg1')]} - self.result.configure_mock(**attrs) - - self.pass_results = mock.Mock() - attrs = {'errors': None, - 'failures': None} - self.pass_results.configure_mock(**attrs) - - def test_run(self): - self.psrunner.case_name = 'test_case_name' + + @mock.patch('unittest.TestLoader') + def _test_run(self, mock_class=None, result=mock.Mock(), + status=testcase.TestCase.EX_OK): with mock.patch('functest.core.pytest_suite_runner.' 'unittest.TextTestRunner.run', - return_value=self.result): - self.assertEqual(self.psrunner.run(), - testcase.TestCase.EX_OK) + return_value=result): + self.assertEqual(self.psrunner.run(), status) + mock_class.assert_not_called() + + def test_check_suite_null(self): + self.assertEqual(self.psrunner.suite, None) + + def test_run_no_ut(self): + mock_result = mock.Mock(testsRun=0, errors=[], failures=[]) + self._test_run(result=mock_result, + status=testcase.TestCase.EX_RUN_ERROR) + self.assertEqual(self.psrunner.result, 0) + self.assertEqual(self.psrunner.details, {'errors': [], 'failures': []}) + self.assertEqual(self.psrunner.is_successful(), + testcase.TestCase.EX_TESTCASE_FAILED) + + def test_run_result_ko(self): + self.psrunner.criteria = 100 + mock_result = mock.Mock(testsRun=50, errors=[('test1', 'error_msg1')], + failures=[('test2', 'failure_msg1')]) + self._test_run(result=mock_result) + self.assertEqual(self.psrunner.result, 96) + self.assertEqual(self.psrunner.details, + {'errors': [('test1', 'error_msg1')], + 'failures': [('test2', 'failure_msg1')]}) + self.assertEqual(self.psrunner.is_successful(), + testcase.TestCase.EX_TESTCASE_FAILED) + + def test_run_result_ok(self): + mock_result = mock.Mock(testsRun=50, errors=[], + failures=[]) + self._test_run(result=mock_result) + self.assertEqual(self.psrunner.result, 100) + self.assertEqual(self.psrunner.details, {'errors': [], 'failures': []}) + self.assertEqual(self.psrunner.is_successful(), + testcase.TestCase.EX_OK) + + @mock.patch('unittest.TestLoader') + def test_run_name_exc(self, mock_class=None): + mock_obj = mock.Mock(side_effect=ImportError) + mock_class.side_effect = mock_obj + self.assertEqual(self.psrunner.run(name='foo'), + testcase.TestCase.EX_RUN_ERROR) + mock_class.assert_called_once_with() + mock_obj.assert_called_once_with() + @mock.patch('unittest.TestLoader') + def test_run_name(self, mock_class=None): + mock_result = mock.Mock(testsRun=50, errors=[], + failures=[]) + mock_obj = mock.Mock() + mock_class.side_effect = mock_obj with mock.patch('functest.core.pytest_suite_runner.' 'unittest.TextTestRunner.run', - return_value=self.pass_results): - self.assertEqual(self.psrunner.run(), + return_value=mock_result): + self.assertEqual(self.psrunner.run(name='foo'), testcase.TestCase.EX_OK) + mock_class.assert_called_once_with() + mock_obj.assert_called_once_with() if __name__ == "__main__": diff --git a/functest/tests/unit/openstack/refstack_client/test_refstack_client.py b/functest/tests/unit/openstack/refstack_client/test_refstack_client.py index 58ec5a07..8c149baa 100644 --- a/functest/tests/unit/openstack/refstack_client/test_refstack_client.py +++ b/functest/tests/unit/openstack/refstack_client/test_refstack_client.py @@ -17,10 +17,12 @@ from functest.utils.constants import CONST class OSRefstackClientTesting(unittest.TestCase): - _config = os.path.join(CONST.dir_functest_test, - CONST.refstack_tempest_conf_path) - _testlist = os.path.join(CONST.dir_functest_test, - CONST.refstack_defcore_list) + _config = os.path.join( + CONST.__getattribute__('dir_functest_test'), + CONST.__getattribute__('refstack_tempest_conf_path')) + _testlist = os.path.join( + CONST.__getattribute__('dir_functest_test'), + CONST.__getattribute__('refstack_defcore_list')) def setUp(self): self.defaultargs = {'config': self._config, @@ -28,12 +30,13 @@ class OSRefstackClientTesting(unittest.TestCase): self.refstackclient = refstack_client.RefstackClient() def test_source_venv(self): - CONST.dir_refstack_client = 'test_repo_dir' + CONST.__setattr__('dir_refstack_client', 'test_repo_dir') with mock.patch('functest.opnfv_tests.openstack.refstack_client.' 'refstack_client.ft_utils.execute_command') as m: cmd = ("cd {0};" ". .venv/bin/activate;" - "cd -;".format(CONST.dir_refstack_client)) + "cd -;" + .format(CONST.__getattribute__('dir_refstack_client'))) self.refstackclient.source_venv() m.assert_any_call(cmd) @@ -44,9 +47,10 @@ class OSRefstackClientTesting(unittest.TestCase): 'refstack_client.ft_utils.execute_command') as m: cmd = ("cd {0};" "./refstack-client test -c {1} -v --test-list {2};" - "cd -;".format(CONST.dir_refstack_client, - config, - testlist)) + "cd -;" + .format(CONST.__getattribute__('dir_refstack_client'), + config, + testlist)) self.refstackclient.run_defcore(config, testlist) m.assert_any_call(cmd) @@ -62,7 +66,7 @@ class OSRefstackClientTesting(unittest.TestCase): self.assertEqual(self.refstackclient.main(**kwargs), status) if len(args) > 0: args[0].assert_called_once_with( - refstack_client.RefstackClient.result_dir) + refstack_client.RefstackClient.result_dir) if len(args) > 1: args |