diff options
| -rw-r--r-- | docker/smoke/testcases.yaml | 3 | ||||
| -rw-r--r-- | functest/ci/testcases.yaml | 3 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/tempest/custom_tests/public_blacklist.yaml | 15 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/tempest/tempest.py | 16 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/tempest/test_tempest.py | 6 | ||||
| -rw-r--r-- | functest/utils/env.py | 3 |
6 files changed, 40 insertions, 6 deletions
diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml index 00930a5bc..e600abd3f 100644 --- a/docker/smoke/testcases.yaml +++ b/docker/smoke/testcases.yaml @@ -44,6 +44,7 @@ tiers: criteria: 100 blocking: false deny_skipping: true + tests_count: 219 description: >- This test case runs a sub group of tests of the OpenStack Compute testcases. @@ -60,6 +61,7 @@ tiers: criteria: 100 blocking: false deny_skipping: true + tests_count: 57 description: >- This test case runs a sub group of tests of the OpenStack Object testcases. @@ -76,6 +78,7 @@ tiers: criteria: 100 blocking: false deny_skipping: true + tests_count: 275 description: >- This test case runs a sub group of tests of the OpenStack Platform testcases. diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 8b2a6c345..f7d8e9bdf 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -207,6 +207,7 @@ tiers: criteria: 100 blocking: false deny_skipping: true + tests_count: 219 description: >- This test case runs a sub group of tests of the OpenStack Compute testcases. @@ -223,6 +224,7 @@ tiers: criteria: 100 blocking: false deny_skipping: true + tests_count: 57 description: >- This test case runs a sub group of tests of the OpenStack Object testcases. @@ -239,6 +241,7 @@ tiers: criteria: 100 blocking: false deny_skipping: true + tests_count: 275 description: >- This test case runs a sub group of tests of the OpenStack Platform testcases. diff --git a/functest/opnfv_tests/openstack/tempest/custom_tests/public_blacklist.yaml b/functest/opnfv_tests/openstack/tempest/custom_tests/public_blacklist.yaml new file mode 100644 index 000000000..e53b577b2 --- /dev/null +++ b/functest/opnfv_tests/openstack/tempest/custom_tests/public_blacklist.yaml @@ -0,0 +1,15 @@ +--- +- + scenarios: + - os-* + tests: + - neutron_tempest_plugin.api.admin.test_floating_ips_admin_actions.FloatingIPAdminTestJSON.test_associate_floating_ip_with_port_from_another_project + - neutron_tempest_plugin.api.admin.test_quotas.QuotasTest.test_detail_quotas + - neutron_tempest_plugin.api.admin.test_quotas.QuotasTest.test_quotas + - neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_floatingip_when_quotas_is_full + - neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_network_when_quotas_is_full + - neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_port_when_quotas_is_full + - neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_router_when_quotas_is_full + - neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_security_group_rule_when_quotas_is_full + - neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_security_group_when_quotas_is_full + - neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_subnet_when_quotas_is_full diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index 908d3bc16..d30d59468 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -50,6 +50,9 @@ class TempestCommon(singlevm.VmReady2): tempest_blacklist = pkg_resources.resource_filename( 'functest', 'opnfv_tests/openstack/tempest/custom_tests/blacklist.yaml') + tempest_public_blacklist = pkg_resources.resource_filename( + 'functest', + 'opnfv_tests/openstack/tempest/custom_tests/public_blacklist.yaml') def __init__(self, **kwargs): if "case_name" not in kwargs: @@ -95,6 +98,7 @@ class TempestCommon(singlevm.VmReady2): except Exception: # pylint: disable=broad-except pass self.deny_skipping = kwargs.get("deny_skipping", False) + self.tests_count = kwargs.get("tests_count", 0) def check_services(self): """Check the mandatory services.""" @@ -350,7 +354,7 @@ class TempestCommon(singlevm.VmReady2): LOGGER.info("%s\n%s", cmd, output.decode("utf-8")) os.remove('/etc/tempest.conf') - def apply_tempest_blacklist(self): + def apply_tempest_blacklist(self, black_list): """Exclude blacklisted test cases.""" LOGGER.debug("Applying tempest blacklist...") if os.path.exists(self.raw_list): @@ -363,7 +367,7 @@ class TempestCommon(singlevm.VmReady2): deploy_scenario = env.get('DEPLOY_SCENARIO') if bool(deploy_scenario): # if DEPLOY_SCENARIO is set we read the file - black_list_file = open(self.tempest_blacklist) + black_list_file = open(black_list) black_list_yaml = yaml.safe_load(black_list_file) black_list_file.close() for item in black_list_yaml: @@ -602,7 +606,10 @@ class TempestCommon(singlevm.VmReady2): shutil.copy("/etc/rally/rally.conf", self.res_dir) self.configure(**kwargs) self.generate_test_list(**kwargs) - self.apply_tempest_blacklist() + self.apply_tempest_blacklist(TempestCommon.tempest_blacklist) + if env.get('PUBLIC_ENDPOINT_ONLY').lower() == 'true': + self.apply_tempest_blacklist( + TempestCommon.tempest_public_blacklist) self.run_verifier_tests(**kwargs) self.parse_verifier_result() rally.RallyBase.verify_report( @@ -636,6 +643,9 @@ class TempestCommon(singlevm.VmReady2): skips = self.details.get("skipped_number", 0) if skips > 0 and self.deny_skipping: return testcase.TestCase.EX_TESTCASE_FAILED + if self.tests_count and ( + self.details.get("tests_number", 0) != self.tests_count): + return testcase.TestCase.EX_TESTCASE_FAILED return super(TempestCommon, self).is_successful() diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py index 87438ae7c..71aa5a257 100644 --- a/functest/tests/unit/openstack/tempest/test_tempest.py +++ b/functest/tests/unit/openstack/tempest/test_tempest.py @@ -125,7 +125,8 @@ class OSTempestTesting(unittest.TestCase): return_value=['test1', 'test2']): self.tempestcommon.tempest_blacklist = Exception os.environ['DEPLOY_SCENARIO'] = 'deploy_scenario' - self.tempestcommon.apply_tempest_blacklist() + self.tempestcommon.apply_tempest_blacklist( + self.tempestcommon.tempest_blacklist) obj = mock_open() obj.write.assert_any_call('test1\n') obj.write.assert_any_call('test2\n') @@ -147,7 +148,8 @@ class OSTempestTesting(unittest.TestCase): mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'yaml.safe_load', return_value=item_dict): os.environ['DEPLOY_SCENARIO'] = 'deploy_scenario' - self.tempestcommon.apply_tempest_blacklist() + self.tempestcommon.apply_tempest_blacklist( + self.tempestcommon.tempest_blacklist) obj = mock_open() obj.write.assert_any_call('test1\n') self.assertFalse(obj.write.assert_any_call('test2\n')) diff --git a/functest/utils/env.py b/functest/utils/env.py index dec9b0e71..85676ed57 100644 --- a/functest/utils/env.py +++ b/functest/utils/env.py @@ -37,7 +37,8 @@ INPUTS = { 'USE_DYNAMIC_CREDENTIALS': 'True', 'BLOCK_MIGRATION': 'True', 'CLEAN_ORPHAN_SECURITY_GROUPS': 'True', - 'SKIP_DOWN_HYPERVISORS': 'False' + 'SKIP_DOWN_HYPERVISORS': 'False', + 'PUBLIC_ENDPOINT_ONLY': 'False' } |