diff options
Diffstat (limited to 'functest')
| -rwxr-xr-x | functest/ci/run_tests.py | 15 | ||||
| -rw-r--r-- | functest/ci/testcases.yaml | 1 | ||||
| -rw-r--r-- | functest/ci/tier_builder.py | 5 | ||||
| -rw-r--r-- | functest/core/pytest_suite_runner.py | 42 | ||||
| -rwxr-xr-x | functest/opnfv_tests/openstack/refstack_client/refstack_client.py | 24 | ||||
| -rwxr-xr-x | functest/opnfv_tests/openstack/refstack_client/tempest_conf.py | 8 | ||||
| -rwxr-xr-x | functest/opnfv_tests/sdn/odl/odl.py | 2 | ||||
| -rw-r--r-- | functest/tests/unit/core/test_pytest_suite_runner.py | 76 | ||||
| -rw-r--r-- | functest/tests/unit/core/test_vnf.py | 4 | ||||
| -rw-r--r-- | functest/tests/unit/features/test_barometer.py | 5 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/refstack_client/test_refstack_client.py | 24 | ||||
| -rw-r--r-- | functest/tests/unit/test_logging.ini | 27 | ||||
| -rw-r--r-- | functest/tests/unit/utils/test_decorators.py | 2 | ||||
| -rw-r--r-- | functest/tests/unit/utils/test_functest_utils.py | 3 | ||||
| -rw-r--r-- | functest/utils/env.py | 4 | ||||
| -rw-r--r-- | functest/utils/functest_utils.py | 8 | ||||
| -rw-r--r-- | functest/utils/openstack_utils.py | 8 |
17 files changed, 134 insertions, 124 deletions
diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py index f973b616..95353c87 100755 --- a/functest/ci/run_tests.py +++ b/functest/ci/run_tests.py @@ -122,10 +122,9 @@ class Runner(object): if not test.is_enabled(): raise TestNotEnabled( "The test case {} is not enabled".format(test.get_name())) - test_name = test.get_name() logger.info("\n") # blank line self.print_separator("=") - logger.info("Running test case '%s'..." % test_name) + logger.info("Running test case '%s'..." % test.get_name()) self.print_separator("=") logger.debug("\n%s" % test) self.source_rc_file() @@ -133,17 +132,17 @@ class Runner(object): if test.needs_clean() and self.clean_flag: self.generate_os_snapshot() - flags = (" -t %s" % (test_name)) + flags = " -t %s" % test.get_name() if self.report_flag: flags += " -r" result = testcase.TestCase.EX_RUN_ERROR - run_dict = self.get_run_dict(test_name) + run_dict = self.get_run_dict(test.get_name()) if run_dict: try: module = importlib.import_module(run_dict['module']) cls = getattr(module, run_dict['class']) - test_dict = ft_utils.get_dict_by_test(test_name) + test_dict = ft_utils.get_dict_by_test(test.get_name()) test_case = cls(**test_dict) self.executed_test_cases.append(test_case) try: @@ -168,7 +167,7 @@ class Runner(object): if test.needs_clean() and self.clean_flag: self.cleanup() if result != testcase.TestCase.EX_OK: - logger.error("The test case '%s' failed. " % test_name) + logger.error("The test case '%s' failed. " % test.get_name()) self.overall_result = Result.EX_ERROR if test.is_blocking(): raise BlockingTestFailed( @@ -242,8 +241,10 @@ class Runner(object): return Result.EX_ERROR else: self.run_all(_tiers) + except BlockingTestFailed: + pass except Exception: - logger.exception("Runner failed") + logger.exception("Failures when running testcase(s)") self.overall_result = Result.EX_ERROR msg = prettytable.PrettyTable( diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index d98a2de2..10587f26 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -302,7 +302,6 @@ tiers: - case_name: bgpvpn - enabled: false project_name: sdnvpn criteria: 100 blocking: false diff --git a/functest/ci/tier_builder.py b/functest/ci/tier_builder.py index 12562f09..d03cd903 100644 --- a/functest/ci/tier_builder.py +++ b/functest/ci/tier_builder.py @@ -54,8 +54,9 @@ class TierBuilder(object): blocking=dic_testcase['blocking'], clean_flag=dic_testcase['clean_flag'], description=dic_testcase['description']) - if (testcase.is_compatible(self.ci_installer, self.ci_scenario) - and testcase.is_enabled()): + if (testcase.is_compatible(self.ci_installer, + self.ci_scenario) and + testcase.is_enabled()): tier.add_test(testcase) self.tier_objects.append(tier) diff --git a/functest/core/pytest_suite_runner.py b/functest/core/pytest_suite_runner.py index a6e47660..efcef7b6 100644 --- a/functest/core/pytest_suite_runner.py +++ b/functest/core/pytest_suite_runner.py @@ -7,6 +7,8 @@ # pylint: disable=missing-docstring +from __future__ import division + import logging import time import unittest @@ -46,32 +48,14 @@ class PyTestSuiteRunner(testcase.TestCase): stream=stream, verbosity=2).run(self.suite) self.logger.debug("\n\n%s", stream.getvalue()) self.stop_time = time.time() - - if result.errors: - self.logger.error('Number of errors in test suite - ' + - str(len(result.errors))) - for test, message in result.errors: - self.logger.error(str(test) + " ERROR with " + message) - - if result.failures: - self.logger.error('Number of failures in test suite - ' + - str(len(result.failures))) - for test, message in result.failures: - self.logger.error(str(test) + " FAILED with " + message) - - # a result can be PASS or FAIL - # But in this case it means that the Execution was OK - # we shall distinguish Execution Error from FAIL results - # TestCase.EX_RUN_ERROR means that the test case was not run - # not that it was run but the result was FAIL - exit_code = testcase.TestCase.EX_OK - if ((result.errors and len(result.errors) > 0) or - (result.failures and len(result.failures) > 0)): - self.logger.info("%s FAILED", self.case_name) - self.result = 0 - else: - self.logger.info("%s OK", self.case_name) - self.result = 100 - - self.details = {} - return exit_code + self.details = {"failures": result.failures, + "errors": result.errors} + try: + self.result = 100 * ( + (result.testsRun - (len(result.failures) + + len(result.errors))) / + result.testsRun) + return testcase.TestCase.EX_OK + except ZeroDivisionError: + self.logger.error("No test has been run") + return testcase.TestCase.EX_RUN_ERROR diff --git a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py index 2a2718dd..5f1f3a1d 100755 --- a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py +++ b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py @@ -33,9 +33,9 @@ class RefstackClient(testcase.TestCase): if "case_name" not in kwargs: kwargs["case_name"] = "refstack_defcore" super(RefstackClient, self).__init__(**kwargs) - self.FUNCTEST_TEST = CONST.dir_functest_test - self.CONF_PATH = CONST.refstack_tempest_conf_path - self.DEFCORE_LIST = CONST.refstack_defcore_list + self.FUNCTEST_TEST = CONST.__getattribute__('dir_functest_test') + self.CONF_PATH = CONST.__getattribute__('refstack_tempest_conf_path') + self.DEFCORE_LIST = CONST.__getattribute__('refstack_defcore_list') self.confpath = os.path.join(self.FUNCTEST_TEST, self.CONF_PATH) self.defcorelist = os.path.join(self.FUNCTEST_TEST, @@ -45,7 +45,7 @@ class RefstackClient(testcase.TestCase): cmd = ("cd {0};" ". .venv/bin/activate;" - "cd -;".format(CONST.dir_refstack_client)) + "cd -;".format(CONST.__getattribute__('dir_refstack_client'))) ft_utils.execute_command(cmd) def run_defcore(self, conf, testlist): @@ -53,7 +53,7 @@ class RefstackClient(testcase.TestCase): cmd = ("cd {0};" "./refstack-client test -c {1} -v --test-list {2};" - "cd -;".format(CONST.dir_refstack_client, + "cd -;".format(CONST.__getattribute__('dir_refstack_client'), conf, testlist)) ft_utils.execute_command(cmd) @@ -63,16 +63,16 @@ class RefstackClient(testcase.TestCase): cmd = ("cd {0};" "./refstack-client test -c {1} -v --test-list {2};" - "cd -;".format(CONST.dir_refstack_client, + "cd -;".format(CONST.__getattribute__('dir_refstack_client'), self.confpath, self.defcorelist)) logger.info("Starting Refstack_defcore test case: '%s'." % cmd) header = ("Refstack environment:\n" " SUT: %s\n Scenario: %s\n Node: %s\n Date: %s\n" % - (CONST.INSTALLER_TYPE, - CONST.DEPLOY_SCENARIO, - CONST.NODE_NAME, + (CONST.__getattribute__('INSTALLER_TYPE'), + CONST.__getattribute__('DEPLOY_SCENARIO'), + CONST.__getattribute__('NODE_NAME'), time.strftime("%a %b %d %H:%M:%S %Z %Y"))) f_stdout = open( @@ -202,9 +202,9 @@ class RefstackClient(testcase.TestCase): class RefstackClientParser(object): def __init__(self): - self.FUNCTEST_TEST = CONST.dir_functest_test - self.CONF_PATH = CONST.refstack_tempest_conf_path - self.DEFCORE_LIST = CONST.refstack_defcore_list + self.FUNCTEST_TEST = CONST.__getattribute__('dir_functest_test') + self.CONF_PATH = CONST.__getattribute__('refstack_tempest_conf_path') + self.DEFCORE_LIST = CONST.__getattribute__('refstack_defcore_list') self.confpath = os.path.join(self.FUNCTEST_TEST, self.CONF_PATH) self.defcorelist = os.path.join(self.FUNCTEST_TEST, diff --git a/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py b/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py index 5c04253c..fbaad589 100755 --- a/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py +++ b/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py @@ -24,12 +24,14 @@ class TempestConf(object): self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id() self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir( self.VERIFIER_ID, self.DEPLOYMENT_ID) - self.confpath = os.path.join(CONST.dir_functest_test, - CONST.refstack_tempest_conf_path) + self.confpath = os.path.join( + CONST.__getattribute__('dir_functest_test'), + CONST.__getattribute__('refstack_tempest_conf_path')) def generate_tempestconf(self): try: - openstack_utils.source_credentials(CONST.openstack_creds) + openstack_utils.source_credentials( + CONST.__getattribute__('openstack_creds')) img_flavor_dict = conf_utils.create_tempest_resources( use_custom_images=True, use_custom_flavors=True) conf_utils.configure_tempest_defcore( diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py index 2f3dd74b..b2b0b77c 100755 --- a/functest/opnfv_tests/sdn/odl/odl.py +++ b/functest/opnfv_tests/sdn/odl/odl.py @@ -108,7 +108,7 @@ class ODLTests(testcase.TestCase): result.suite.statistics.critical.passed / result.suite.statistics.critical.total) except ZeroDivisionError: - self.__logger.error("No test has been ran") + self.__logger.error("No test has been run") self.start_time = timestamp_to_secs(result.suite.starttime) self.stop_time = timestamp_to_secs(result.suite.endtime) self.details = {} diff --git a/functest/tests/unit/core/test_pytest_suite_runner.py b/functest/tests/unit/core/test_pytest_suite_runner.py index 07ac7906..f317cdea 100644 --- a/functest/tests/unit/core/test_pytest_suite_runner.py +++ b/functest/tests/unit/core/test_pytest_suite_runner.py @@ -20,29 +20,71 @@ class PyTestSuiteRunnerTesting(unittest.TestCase): def setUp(self): self.psrunner = pytest_suite_runner.PyTestSuiteRunner() - self.result = mock.Mock() - attrs = {'errors': [('test1', 'error_msg1')], - 'failures': [('test2', 'failure_msg1')]} - self.result.configure_mock(**attrs) - - self.pass_results = mock.Mock() - attrs = {'errors': None, - 'failures': None} - self.pass_results.configure_mock(**attrs) - - def test_run(self): - self.psrunner.case_name = 'test_case_name' + + @mock.patch('unittest.TestLoader') + def _test_run(self, mock_class=None, result=mock.Mock(), + status=testcase.TestCase.EX_OK): with mock.patch('functest.core.pytest_suite_runner.' 'unittest.TextTestRunner.run', - return_value=self.result): - self.assertEqual(self.psrunner.run(), - testcase.TestCase.EX_OK) + return_value=result): + self.assertEqual(self.psrunner.run(), status) + mock_class.assert_not_called() + + def test_check_suite_null(self): + self.assertEqual(self.psrunner.suite, None) + + def test_run_no_ut(self): + mock_result = mock.Mock(testsRun=0, errors=[], failures=[]) + self._test_run(result=mock_result, + status=testcase.TestCase.EX_RUN_ERROR) + self.assertEqual(self.psrunner.result, 0) + self.assertEqual(self.psrunner.details, {'errors': [], 'failures': []}) + self.assertEqual(self.psrunner.is_successful(), + testcase.TestCase.EX_TESTCASE_FAILED) + + def test_run_result_ko(self): + self.psrunner.criteria = 100 + mock_result = mock.Mock(testsRun=50, errors=[('test1', 'error_msg1')], + failures=[('test2', 'failure_msg1')]) + self._test_run(result=mock_result) + self.assertEqual(self.psrunner.result, 96) + self.assertEqual(self.psrunner.details, + {'errors': [('test1', 'error_msg1')], + 'failures': [('test2', 'failure_msg1')]}) + self.assertEqual(self.psrunner.is_successful(), + testcase.TestCase.EX_TESTCASE_FAILED) + + def test_run_result_ok(self): + mock_result = mock.Mock(testsRun=50, errors=[], + failures=[]) + self._test_run(result=mock_result) + self.assertEqual(self.psrunner.result, 100) + self.assertEqual(self.psrunner.details, {'errors': [], 'failures': []}) + self.assertEqual(self.psrunner.is_successful(), + testcase.TestCase.EX_OK) + + @mock.patch('unittest.TestLoader') + def test_run_name_exc(self, mock_class=None): + mock_obj = mock.Mock(side_effect=ImportError) + mock_class.side_effect = mock_obj + self.assertEqual(self.psrunner.run(name='foo'), + testcase.TestCase.EX_RUN_ERROR) + mock_class.assert_called_once_with() + mock_obj.assert_called_once_with() + @mock.patch('unittest.TestLoader') + def test_run_name(self, mock_class=None): + mock_result = mock.Mock(testsRun=50, errors=[], + failures=[]) + mock_obj = mock.Mock() + mock_class.side_effect = mock_obj with mock.patch('functest.core.pytest_suite_runner.' 'unittest.TextTestRunner.run', - return_value=self.pass_results): - self.assertEqual(self.psrunner.run(), + return_value=mock_result): + self.assertEqual(self.psrunner.run(name='foo'), testcase.TestCase.EX_OK) + mock_class.assert_called_once_with() + mock_obj.assert_called_once_with() if __name__ == "__main__": diff --git a/functest/tests/unit/core/test_vnf.py b/functest/tests/unit/core/test_vnf.py index e322773e..ce859040 100644 --- a/functest/tests/unit/core/test_vnf.py +++ b/functest/tests/unit/core/test_vnf.py @@ -146,12 +146,12 @@ class VnfBaseTesting(unittest.TestCase): def test_deploy_vnf_unimplemented(self): with self.assertRaises(Exception) as context: self.test.deploy_vnf() - self.assertTrue('VNF not deployed' in context.exception) + self.assertIn('VNF not deployed', str(context.exception)) def test_test_vnf_unimplemented(self): with self.assertRaises(Exception) as context: self.test.test_vnf()() - self.assertTrue('VNF not tested' in context.exception) + self.assertIn('VNF not tested', str(context.exception)) def test_parse_results_ex_ok(self): self.test.details['test_vnf']['status'] = 'PASS' diff --git a/functest/tests/unit/features/test_barometer.py b/functest/tests/unit/features/test_barometer.py index c6512615..8c2585d9 100644 --- a/functest/tests/unit/features/test_barometer.py +++ b/functest/tests/unit/features/test_barometer.py @@ -16,9 +16,8 @@ import unittest import mock from functest.core import testcase -sys.modules['baro_tests'] = mock.Mock() # noqa -# pylint: disable=wrong-import-position -from functest.opnfv_tests.features import barometer +with mock.patch('functest.utils.functest_utils.get_parameter_from_yaml'): + from functest.opnfv_tests.features import barometer class BarometerTesting(unittest.TestCase): diff --git a/functest/tests/unit/openstack/refstack_client/test_refstack_client.py b/functest/tests/unit/openstack/refstack_client/test_refstack_client.py index 58ec5a07..8c149baa 100644 --- a/functest/tests/unit/openstack/refstack_client/test_refstack_client.py +++ b/functest/tests/unit/openstack/refstack_client/test_refstack_client.py @@ -17,10 +17,12 @@ from functest.utils.constants import CONST class OSRefstackClientTesting(unittest.TestCase): - _config = os.path.join(CONST.dir_functest_test, - CONST.refstack_tempest_conf_path) - _testlist = os.path.join(CONST.dir_functest_test, - CONST.refstack_defcore_list) + _config = os.path.join( + CONST.__getattribute__('dir_functest_test'), + CONST.__getattribute__('refstack_tempest_conf_path')) + _testlist = os.path.join( + CONST.__getattribute__('dir_functest_test'), + CONST.__getattribute__('refstack_defcore_list')) def setUp(self): self.defaultargs = {'config': self._config, @@ -28,12 +30,13 @@ class OSRefstackClientTesting(unittest.TestCase): self.refstackclient = refstack_client.RefstackClient() def test_source_venv(self): - CONST.dir_refstack_client = 'test_repo_dir' + CONST.__setattr__('dir_refstack_client', 'test_repo_dir') with mock.patch('functest.opnfv_tests.openstack.refstack_client.' 'refstack_client.ft_utils.execute_command') as m: cmd = ("cd {0};" ". .venv/bin/activate;" - "cd -;".format(CONST.dir_refstack_client)) + "cd -;" + .format(CONST.__getattribute__('dir_refstack_client'))) self.refstackclient.source_venv() m.assert_any_call(cmd) @@ -44,9 +47,10 @@ class OSRefstackClientTesting(unittest.TestCase): 'refstack_client.ft_utils.execute_command') as m: cmd = ("cd {0};" "./refstack-client test -c {1} -v --test-list {2};" - "cd -;".format(CONST.dir_refstack_client, - config, - testlist)) + "cd -;" + .format(CONST.__getattribute__('dir_refstack_client'), + config, + testlist)) self.refstackclient.run_defcore(config, testlist) m.assert_any_call(cmd) @@ -62,7 +66,7 @@ class OSRefstackClientTesting(unittest.TestCase): self.assertEqual(self.refstackclient.main(**kwargs), status) if len(args) > 0: args[0].assert_called_once_with( - refstack_client.RefstackClient.result_dir) + refstack_client.RefstackClient.result_dir) if len(args) > 1: args diff --git a/functest/tests/unit/test_logging.ini b/functest/tests/unit/test_logging.ini deleted file mode 100644 index 492767d1..00000000 --- a/functest/tests/unit/test_logging.ini +++ /dev/null @@ -1,27 +0,0 @@ -[loggers] -keys=root,functest_logger - -[logger_root] -level=DEBUG -handlers=console - -[logger_functest_logger] -level=DEBUG -handlers=console -qualname=functest.utils.functest_logger -propagate=0 - -[handlers] -keys=console - -[handler_console] -class=StreamHandler -level=INFO -formatter=standard -args=(sys.stdout,) - -[formatters] -keys=standard - -[formatter_standard] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
\ No newline at end of file diff --git a/functest/tests/unit/utils/test_decorators.py b/functest/tests/unit/utils/test_decorators.py index 6bd47d25..44448f23 100644 --- a/functest/tests/unit/utils/test_decorators.py +++ b/functest/tests/unit/utils/test_decorators.py @@ -63,7 +63,7 @@ class DecoratorsTesting(unittest.TestCase): 'pod_name': self._node_name, 'installer': self._installer_type, 'scenario': self._deploy_scenario, 'version': VERSION, 'details': {}, 'criteria': self._result} - return json.dumps(data) + return json.dumps(data, sort_keys=True) @mock.patch('{}.get_db_url'.format(functest_utils.__name__), return_value='http://127.0.0.1') diff --git a/functest/tests/unit/utils/test_functest_utils.py b/functest/tests/unit/utils/test_functest_utils.py index 0fe7e91d..218f7f72 100644 --- a/functest/tests/unit/utils/test_functest_utils.py +++ b/functest/tests/unit/utils/test_functest_utils.py @@ -55,7 +55,8 @@ class FunctestUtilsTesting(unittest.TestCase): self.testcase_dict = {'case_name': 'testname', 'criteria': self.criteria} self.parameter = 'general.openstack.image_name' - self.config_yaml = 'test_config_yaml-' + self.config_yaml = os.path.normpath(os.path.join(os.path.dirname( + os.path.abspath(__file__)), '../../../ci/config_functest.yaml')) self.db_url_env = 'http://foo/testdb' self.file_yaml = {'general': {'openstack': {'image_name': 'test_image_name'}}} diff --git a/functest/utils/env.py b/functest/utils/env.py index c9629e15..3724ec99 100644 --- a/functest/utils/env.py +++ b/functest/utils/env.py @@ -15,7 +15,9 @@ default_envs = { 'INSTALLER_IP': None, 'BUILD_TAG': None, 'OS_ENDPOINT_TYPE': None, - 'OS_AUTH_URL': None + 'OS_AUTH_URL': None, + 'CONFIG_FUNCTEST_YAML': os.path.normpath(os.path.join(os.path.dirname( + os.path.abspath(__file__)), '../ci/config_functest.yaml')) } diff --git a/functest/utils/functest_utils.py b/functest/utils/functest_utils.py index bf30f56e..dc20eea1 100644 --- a/functest/utils/functest_utils.py +++ b/functest/utils/functest_utils.py @@ -24,6 +24,7 @@ from six.moves import urllib import yaml from git import Repo +from functest.utils import constants from functest.utils import decorators logger = logging.getLogger(__name__) @@ -220,7 +221,8 @@ def push_results_to_db(project, case_name, error = None headers = {'Content-Type': 'application/json'} try: - r = requests.post(url, data=json.dumps(params), headers=headers) + r = requests.post(url, data=json.dumps(params, sort_keys=True), + headers=headers) logger.debug(r) r.raise_for_status() except requests.RequestException as exc: @@ -375,7 +377,7 @@ def get_parameter_from_yaml(parameter, file): def get_functest_config(parameter): - yaml_ = os.environ["CONFIG_FUNCTEST_YAML"] + yaml_ = constants.CONST.__getattribute__('CONFIG_FUNCTEST_YAML') return get_parameter_from_yaml(parameter, yaml_) @@ -397,7 +399,7 @@ def get_testcases_file_dir(): def get_functest_yaml(): - with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f: + with open(constants.CONST.__getattribute__('CONFIG_FUNCTEST_YAML')) as f: functest_yaml = yaml.safe_load(f) f.close() return functest_yaml diff --git a/functest/utils/openstack_utils.py b/functest/utils/openstack_utils.py index 8bd95052..57a2aa2b 100644 --- a/functest/utils/openstack_utils.py +++ b/functest/utils/openstack_utils.py @@ -1081,10 +1081,10 @@ def check_security_group_rules(neutron_client, sg_id, direction, protocol, try: security_rules = get_security_group_rules(neutron_client, sg_id) security_rules = [rule for rule in security_rules - if (rule["direction"].lower() == direction - and rule["protocol"].lower() == protocol - and rule["port_range_min"] == port_min - and rule["port_range_max"] == port_max)] + if (rule["direction"].lower() == direction and + rule["protocol"].lower() == protocol and + rule["port_range_min"] == port_min and + rule["port_range_max"] == port_max)] if len(security_rules) == 0: return True else: |