aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Kulhavy <martin.kulhavy@nokia.com>2017-07-07 17:39:33 +0300
committerMartin Kulhavy <martin.kulhavy@nokia.com>2017-07-10 12:30:00 +0300
commit053bc5f800de4c5543452f0cf27e26d6e5d9d96b (patch)
tree7619b0653d87a516a8995704fa0f03b2940b4d6d
parent26972ddefa2349dcc1ffe96be25a905046be8110 (diff)
Allow regex for blacklist scenarios/installers
Reasoning: Some blacklisted test cases apply to a range of scenarios, e.g. both 'ha' and 'noha' and possibly even to all scenarios for a specific installer -- therefore the list of excluded scenarios can get long and unclear. For consistency with rally test case scenarios also allow regex for installers specification. Additionally, log the excluded test cases (if any) to debug log. Change-Id: Ie0bec35d1a6443203d4719ea5428c437c12e5e12 Signed-off-by: Martin Kulhavy <martin.kulhavy@nokia.com>
-rw-r--r--functest/opnfv_tests/openstack/rally/blacklist.txt3
-rw-r--r--functest/opnfv_tests/openstack/rally/rally.py44
-rw-r--r--functest/tests/unit/openstack/rally/test_rally.py33
3 files changed, 68 insertions, 12 deletions
diff --git a/functest/opnfv_tests/openstack/rally/blacklist.txt b/functest/opnfv_tests/openstack/rally/blacklist.txt
index 3a17fa61..95bea2b7 100644
--- a/functest/opnfv_tests/openstack/rally/blacklist.txt
+++ b/functest/opnfv_tests/openstack/rally/blacklist.txt
@@ -1,8 +1,7 @@
scenario:
-
scenarios:
- - os-nosdn-lxd-ha
- - os-nosdn-lxd-noha
+ - '^os-nosdn-lxd-(no)?ha$'
installers:
- joid
tests:
diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py
index 24c9147c..d5acb1b2 100644
--- a/functest/opnfv_tests/openstack/rally/rally.py
+++ b/functest/opnfv_tests/openstack/rally/rally.py
@@ -188,21 +188,44 @@ class RallyBase(testcase.OSGCTestCase):
installer_type = CONST.__getattribute__('INSTALLER_TYPE')
deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
- if (bool(installer_type) * bool(deploy_scenario)):
- if 'scenario' in black_list_yaml.keys():
- for item in black_list_yaml['scenario']:
- scenarios = item['scenarios']
- installers = item['installers']
- if (deploy_scenario in scenarios and
- installer_type in installers):
- tests = item['tests']
- black_tests.extend(tests)
+ if (bool(installer_type) and bool(deploy_scenario) and
+ 'scenario' in black_list_yaml.keys()):
+ for item in black_list_yaml['scenario']:
+ scenarios = item['scenarios']
+ installers = item['installers']
+ in_it = RallyBase.in_iterable_re
+ if (in_it(deploy_scenario, scenarios) and
+ in_it(installer_type, installers)):
+ tests = item['tests']
+ black_tests.extend(tests)
except Exception:
logger.debug("Scenario exclusion not applied.")
return black_tests
@staticmethod
+ def in_iterable_re(needle, haystack):
+ """
+ Check if given needle is in the iterable haystack, using regex.
+
+ :param needle: string to be matched
+ :param haystack: iterable of strings (optionally regex patterns)
+ :return: True if needle is eqial to any of the elements in haystack,
+ or if a nonempty regex pattern in haystack is found in needle.
+ """
+
+ # match without regex
+ if needle in haystack:
+ return True
+
+ for pattern in haystack:
+ # match if regex pattern is set and found in the needle
+ if pattern and re.search(pattern, needle) is not None:
+ return True
+ else:
+ return False
+
+ @staticmethod
def excl_func():
black_tests = []
func_list = []
@@ -235,6 +258,9 @@ class RallyBase(testcase.OSGCTestCase):
black_tests = list(set(RallyBase.excl_func() +
RallyBase.excl_scenario()))
+ if black_tests:
+ logger.debug("Blacklisted tests: " + str(black_tests))
+
include = True
for cases_line in cases_file:
if include:
diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py
index 8845f660..def9c93b 100644
--- a/functest/tests/unit/openstack/rally/test_rally.py
+++ b/functest/tests/unit/openstack/rally/test_rally.py
@@ -130,7 +130,10 @@ class OSRallyTesting(unittest.TestCase):
CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
dic = {'scenario': [{'scenarios': ['test_scenario'],
'installers': ['test_installer'],
- 'tests': ['test']}]}
+ 'tests': ['test']},
+ {'scenarios': ['other_scenario'],
+ 'installers': ['test_installer'],
+ 'tests': ['other_test']}]}
with mock.patch('__builtin__.open', mock.mock_open()), \
mock.patch('functest.opnfv_tests.openstack.rally.rally.'
'yaml.safe_load',
@@ -138,6 +141,34 @@ class OSRallyTesting(unittest.TestCase):
self.assertEqual(self.rally_base.excl_scenario(),
['test'])
+ def test_excl_scenario_regex(self):
+ CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
+ CONST.__setattr__('DEPLOY_SCENARIO', 'os-ctrlT-featT-modeT')
+ dic = {'scenario': [{'scenarios': ['^os-[^-]+-featT-modeT$'],
+ 'installers': ['test_installer'],
+ 'tests': ['test1']},
+ {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
+ 'installers': ['test_installer'],
+ 'tests': ['test2']},
+ {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
+ 'installers': ['test_installer'],
+ 'tests': ['test3']},
+ {'scenarios': ['^os-'],
+ 'installers': ['test_installer'],
+ 'tests': ['test4']},
+ {'scenarios': ['other_scenario'],
+ 'installers': ['test_installer'],
+ 'tests': ['test0a']},
+ {'scenarios': [''], # empty scenario
+ 'installers': ['test_installer'],
+ 'tests': ['test0b']}]}
+ with mock.patch('__builtin__.open', mock.mock_open()), \
+ mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'yaml.safe_load',
+ return_value=dic):
+ self.assertEqual(self.rally_base.excl_scenario(),
+ ['test1', 'test2', 'test3', 'test4'])
+
def test_excl_scenario_exception(self):
with mock.patch('__builtin__.open', side_effect=Exception):
self.assertEqual(self.rally_base.excl_scenario(),