Merge "Allow regex for blacklist scenarios/installers"

3 files changed, 68 insertions, 12 deletions

diff --git a/functest/opnfv_tests/openstack/rally/blacklist.txt b/functest/opnfv_tests/openstack/rally/blacklist.txt
index 3a17fa616..95bea2b7a 100644
--- a/functest/opnfv_tests/openstack/rally/blacklist.txt
+++ b/functest/opnfv_tests/openstack/rally/blacklist.txt
@@ -1,8 +1,7 @@
 scenario:
     -
         scenarios:
-            - os-nosdn-lxd-ha
-            - os-nosdn-lxd-noha
+            - '^os-nosdn-lxd-(no)?ha$'
         installers:
             - joid
         tests:
diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py
index 24c9147c8..d5acb1b23 100644
--- a/functest/opnfv_tests/openstack/rally/rally.py
+++ b/functest/opnfv_tests/openstack/rally/rally.py
@@ -188,21 +188,44 @@ class RallyBase(testcase.OSGCTestCase):
 
             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
-            if (bool(installer_type) * bool(deploy_scenario)):
-                if 'scenario' in black_list_yaml.keys():
-                    for item in black_list_yaml['scenario']:
-                        scenarios = item['scenarios']
-                        installers = item['installers']
-                        if (deploy_scenario in scenarios and
-                                installer_type in installers):
-                            tests = item['tests']
-                            black_tests.extend(tests)
+            if (bool(installer_type) and bool(deploy_scenario) and
+                    'scenario' in black_list_yaml.keys()):
+                for item in black_list_yaml['scenario']:
+                    scenarios = item['scenarios']
+                    installers = item['installers']
+                    in_it = RallyBase.in_iterable_re
+                    if (in_it(deploy_scenario, scenarios) and
+                            in_it(installer_type, installers)):
+                        tests = item['tests']
+                        black_tests.extend(tests)
         except Exception:
             logger.debug("Scenario exclusion not applied.")
 
         return black_tests
 
     @staticmethod
+    def in_iterable_re(needle, haystack):
+        """
+        Check if given needle is in the iterable haystack, using regex.
+
+        :param needle: string to be matched
+        :param haystack: iterable of strings (optionally regex patterns)
+        :return: True if needle is eqial to any of the elements in haystack,
+                 or if a nonempty regex pattern in haystack is found in needle.
+        """
+
+        # match without regex
+        if needle in haystack:
+            return True
+
+        for pattern in haystack:
+            # match if regex pattern is set and found in the needle
+            if pattern and re.search(pattern, needle) is not None:
+                return True
+        else:
+            return False
+
+    @staticmethod
     def excl_func():
         black_tests = []
         func_list = []
@@ -235,6 +258,9 @@ class RallyBase(testcase.OSGCTestCase):
         black_tests = list(set(RallyBase.excl_func() +
                            RallyBase.excl_scenario()))
 
+        if black_tests:
+            logger.debug("Blacklisted tests: " + str(black_tests))
+
         include = True
         for cases_line in cases_file:
             if include:
diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py
index 8845f6604..def9c93b7 100644
--- a/functest/tests/unit/openstack/rally/test_rally.py
+++ b/functest/tests/unit/openstack/rally/test_rally.py
@@ -130,7 +130,10 @@ class OSRallyTesting(unittest.TestCase):
         CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
         dic = {'scenario': [{'scenarios': ['test_scenario'],
                              'installers': ['test_installer'],
-                             'tests': ['test']}]}
+                             'tests': ['test']},
+                            {'scenarios': ['other_scenario'],
+                             'installers': ['test_installer'],
+                             'tests': ['other_test']}]}
         with mock.patch('__builtin__.open', mock.mock_open()), \
             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
                        'yaml.safe_load',
@@ -138,6 +141,34 @@ class OSRallyTesting(unittest.TestCase):
                 self.assertEqual(self.rally_base.excl_scenario(),
                                  ['test'])
 
+    def test_excl_scenario_regex(self):
+        CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
+        CONST.__setattr__('DEPLOY_SCENARIO', 'os-ctrlT-featT-modeT')
+        dic = {'scenario': [{'scenarios': ['^os-[^-]+-featT-modeT$'],
+                             'installers': ['test_installer'],
+                             'tests': ['test1']},
+                            {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
+                             'installers': ['test_installer'],
+                             'tests': ['test2']},
+                            {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
+                             'installers': ['test_installer'],
+                             'tests': ['test3']},
+                            {'scenarios': ['^os-'],
+                             'installers': ['test_installer'],
+                             'tests': ['test4']},
+                            {'scenarios': ['other_scenario'],
+                             'installers': ['test_installer'],
+                             'tests': ['test0a']},
+                            {'scenarios': [''],  # empty scenario
+                             'installers': ['test_installer'],
+                             'tests': ['test0b']}]}
+        with mock.patch('__builtin__.open', mock.mock_open()), \
+            mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+                       'yaml.safe_load',
+                       return_value=dic):
+            self.assertEqual(self.rally_base.excl_scenario(),
+                             ['test1', 'test2', 'test3', 'test4'])
+
     def test_excl_scenario_exception(self):
         with mock.patch('__builtin__.open', side_effect=Exception):
                 self.assertEqual(self.rally_base.excl_scenario(),