From 053bc5f800de4c5543452f0cf27e26d6e5d9d96b Mon Sep 17 00:00:00 2001 From: Martin Kulhavy Date: Fri, 7 Jul 2017 17:39:33 +0300 Subject: Allow regex for blacklist scenarios/installers Reasoning: Some blacklisted test cases apply to a range of scenarios, e.g. both 'ha' and 'noha' and possibly even to all scenarios for a specific installer -- therefore the list of excluded scenarios can get long and unclear. For consistency with rally test case scenarios also allow regex for installers specification. Additionally, log the excluded test cases (if any) to debug log. Change-Id: Ie0bec35d1a6443203d4719ea5428c437c12e5e12 Signed-off-by: Martin Kulhavy --- functest/opnfv_tests/openstack/rally/blacklist.txt | 3 +- functest/opnfv_tests/openstack/rally/rally.py | 44 +++++++++++++++++----- functest/tests/unit/openstack/rally/test_rally.py | 33 +++++++++++++++- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/functest/opnfv_tests/openstack/rally/blacklist.txt b/functest/opnfv_tests/openstack/rally/blacklist.txt index 3a17fa61..95bea2b7 100644 --- a/functest/opnfv_tests/openstack/rally/blacklist.txt +++ b/functest/opnfv_tests/openstack/rally/blacklist.txt @@ -1,8 +1,7 @@ scenario: - scenarios: - - os-nosdn-lxd-ha - - os-nosdn-lxd-noha + - '^os-nosdn-lxd-(no)?ha$' installers: - joid tests: diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 24c9147c..d5acb1b2 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -188,20 +188,43 @@ class RallyBase(testcase.OSGCTestCase): installer_type = CONST.__getattribute__('INSTALLER_TYPE') deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO') - if (bool(installer_type) * bool(deploy_scenario)): - if 'scenario' in black_list_yaml.keys(): - for item in black_list_yaml['scenario']: - scenarios = item['scenarios'] - installers = item['installers'] - if (deploy_scenario in scenarios and - installer_type in installers): - tests = item['tests'] - black_tests.extend(tests) + if (bool(installer_type) and bool(deploy_scenario) and + 'scenario' in black_list_yaml.keys()): + for item in black_list_yaml['scenario']: + scenarios = item['scenarios'] + installers = item['installers'] + in_it = RallyBase.in_iterable_re + if (in_it(deploy_scenario, scenarios) and + in_it(installer_type, installers)): + tests = item['tests'] + black_tests.extend(tests) except Exception: logger.debug("Scenario exclusion not applied.") return black_tests + @staticmethod + def in_iterable_re(needle, haystack): + """ + Check if given needle is in the iterable haystack, using regex. + + :param needle: string to be matched + :param haystack: iterable of strings (optionally regex patterns) + :return: True if needle is eqial to any of the elements in haystack, + or if a nonempty regex pattern in haystack is found in needle. + """ + + # match without regex + if needle in haystack: + return True + + for pattern in haystack: + # match if regex pattern is set and found in the needle + if pattern and re.search(pattern, needle) is not None: + return True + else: + return False + @staticmethod def excl_func(): black_tests = [] @@ -235,6 +258,9 @@ class RallyBase(testcase.OSGCTestCase): black_tests = list(set(RallyBase.excl_func() + RallyBase.excl_scenario())) + if black_tests: + logger.debug("Blacklisted tests: " + str(black_tests)) + include = True for cases_line in cases_file: if include: diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index 8845f660..def9c93b 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -130,7 +130,10 @@ class OSRallyTesting(unittest.TestCase): CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario') dic = {'scenario': [{'scenarios': ['test_scenario'], 'installers': ['test_installer'], - 'tests': ['test']}]} + 'tests': ['test']}, + {'scenarios': ['other_scenario'], + 'installers': ['test_installer'], + 'tests': ['other_test']}]} with mock.patch('__builtin__.open', mock.mock_open()), \ mock.patch('functest.opnfv_tests.openstack.rally.rally.' 'yaml.safe_load', @@ -138,6 +141,34 @@ class OSRallyTesting(unittest.TestCase): self.assertEqual(self.rally_base.excl_scenario(), ['test']) + def test_excl_scenario_regex(self): + CONST.__setattr__('INSTALLER_TYPE', 'test_installer') + CONST.__setattr__('DEPLOY_SCENARIO', 'os-ctrlT-featT-modeT') + dic = {'scenario': [{'scenarios': ['^os-[^-]+-featT-modeT$'], + 'installers': ['test_installer'], + 'tests': ['test1']}, + {'scenarios': ['^os-ctrlT-[^-]+-modeT$'], + 'installers': ['test_installer'], + 'tests': ['test2']}, + {'scenarios': ['^os-ctrlT-featT-[^-]+$'], + 'installers': ['test_installer'], + 'tests': ['test3']}, + {'scenarios': ['^os-'], + 'installers': ['test_installer'], + 'tests': ['test4']}, + {'scenarios': ['other_scenario'], + 'installers': ['test_installer'], + 'tests': ['test0a']}, + {'scenarios': [''], # empty scenario + 'installers': ['test_installer'], + 'tests': ['test0b']}]} + with mock.patch('__builtin__.open', mock.mock_open()), \ + mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'yaml.safe_load', + return_value=dic): + self.assertEqual(self.rally_base.excl_scenario(), + ['test1', 'test2', 'test3', 'test4']) + def test_excl_scenario_exception(self): with mock.patch('__builtin__.open', side_effect=Exception): self.assertEqual(self.rally_base.excl_scenario(), -- cgit 1.2.3-korg