dovetail tool: refstack testcase integration

JIRA: DOVETAIL-370

Change-Id: I60c9f431358f848e24abcb865c4b2ca8c3b2d843
Signed-off-by: MatthewLi <matthew.lijun@huawei.com>

6 files changed, 314 insertions, 7 deletions

diff --git a/dovetail/compliance/debug.yml b/dovetail/compliance/debug.yml
index 87003c47..847251bd 100644
--- a/dovetail/compliance/debug.yml
+++ b/dovetail/compliance/debug.yml
@@ -6,6 +6,7 @@ debug:
   name: debug
   testcases_list:
     - dovetail.example.tc002
+    - dovetail.defcore.tc001
     - dovetail.ipv6.tc008
     - dovetail.ipv6.tc009
     - dovetail.ipv6.tc018
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index ea8ecedc..b8174bdb 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -20,24 +20,29 @@ testsuite_supported:
   - debug
 # testarea supported, should adjust accordingly
 testarea_supported:
-  - vimops
-  - nfvi
-  - ipv6
+  - defcore
   - example
+  - ipv6
+  - nfvi
+  - vimops
 
 functest_testsuite:
+  - refstack_defcore
   - tempest_smoke_serial
   - tempest_full_parallel
   - rally_sanity
   - promise
 
 functest_testcase:
-  - healthcheck
+  - connection_check
+  - api_check
+  - snaps_health_check
   - vping_ssh
   - vping_userdata
   - doctor
   - copper
   - cloudify_ims
+  - odl
 
 # used for testcase cmd template in jinja2 format
 # we have two variables available now
@@ -61,9 +66,6 @@ validate_input:
   valid_docker_tag:
     - 'stable'
     - 'latest'
-    - 'colorado.1.0'
-    - 'colorado.2.0'
-    - 'colorado.3.0'
     - 'danube.1.0'
     - 'danube.2.0'
     - 'danube.3.0'
diff --git a/dovetail/container.py b/dovetail/container.py
index 878b21cb..9f4edd40 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -205,3 +205,11 @@ class Container(object):
             return (1, 'sub_cmd is empty')
         cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd)
         return dt_utils.exec_cmd(cmd, cls.logger, exit_on_error)
+
+    @classmethod
+    def pre_copy(cls, container_id, src_path, dest_path, exit_on_error=False):
+        if src_path == "" or dest_path == "":
+            return (1, 'src_path or dest_path is empty')
+        cmd = 'sudo docker cp %s %s:%s' % (src_path, container_id, dest_path)
+        cls.logger.debug('execute cmd %s', cmd)
+        return dt_utils.exec_cmd(cmd, cls.logger, exit_on_error)
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index ea58768c..a67a6314 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -45,6 +45,11 @@ class DockerRunner(object):
 
         self.logger.debug('container id:%s', container_id)
 
+        dest_path = self.testcase.pre_copy_dest_path()
+        if dest_path:
+            src_path = self.testcase.mk_src_file()
+            ret, msg = Container.pre_copy(container_id, src_path, dest_path)
+
         if not self.testcase.prepared():
             prepare_failed = False
             cmds = self.testcase.pre_condition()
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index aeeeee68..7f218c47 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -115,6 +115,14 @@ class Testcase(object):
                               self.name())
         return pre_condition
 
+    def pre_copy_dest_path(self):
+        try:
+            pre_copy_dest_path = \
+                self.testcase['validate']['pre_copy']['dest_path']
+        except KeyError:
+            pre_copy_dest_path = ''
+        return pre_copy_dest_path
+
     def post_condition(self):
         try:
             post_condition = self.testcase['validate']['post_condition']
@@ -128,6 +136,23 @@ class Testcase(object):
                               self.name())
         return post_condition
 
+    def mk_src_file(self):
+        testcase_src_file = self.testcase['validate']['pre_copy']['src_file']
+        try:
+            with open(os.path.join(dt_cfg.dovetail_config['result_dir'],
+                      testcase_src_file), 'w+') as src_file:
+                if self.sub_testcase() is not None:
+                    for sub_test in self.sub_testcase():
+                        self.logger.info('save testcases %s', sub_test)
+                        src_file.write(sub_test + '\n')
+            self.logger.info('save testcases to %s', src_file)
+        except Exception:
+            self.logger.error('Failed to save: %s', src_file)
+
+        src_file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
+                                     testcase_src_file)
+        return src_file_path
+
     def run(self):
         runner = TestRunnerFactory.create(self)
         try:
diff --git a/dovetail/testcase/defcore.tc001.yml b/dovetail/testcase/defcore.tc001.yml
new file mode 100644
index 00000000..bbd2c313
--- /dev/null
+++ b/dovetail/testcase/defcore.tc001.yml
@@ -0,0 +1,266 @@
+---
+dovetail.defcore.tc001:
+  name: dovetail.defcore.tc001
+  objective: > # Set of DefCore tempest test cases not flagged and required. It only contains OpenStack core (no object storage)
+    # The approved guidelines (2016.08) are valid for Kilo, Liberty, Mitaka and Newton releases of OpenStack
+    # The list can be generated using the Rest API from RefStack project:
+    # https://refstack.openstack.org/api/v1/guidelines/2016.08/tests?target=compute&type=required&alias=true&flag=false
+  validate:
+    type: functest
+    testcase: refstack_defcore
+    pre_condition:
+      - 'echo test for precondition in testcase'
+    pre_copy:
+      src_file: defcore.txt
+      dest_path: /home/opnfv/repos/functest/functest/opnfv_tests/openstack/refstack_client
+    cmds:
+      - 'functest env prepare'
+      - 'functest testcase run refstack_defcore -r'
+    post_condition:
+      - 'echo test for postcondition in testcase'
+  report:
+    sub_testcase_list:
+      - tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image
+      - tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name
+      - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name
+      - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers
+      - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail
+      - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus
+      - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details
+      - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name
+      - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers
+      - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail
+      - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus
+      - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details
+      - tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action
+      - tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_active_status
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status
+      - tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing
+      - tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server
+      - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server
+      - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard
+      - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server
+      - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server
+      - tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_delete_server_metadata_item
+      - tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_get_server_metadata_item
+      - tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_list_server_metadata
+      - tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata
+      - tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata_item
+      - tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_server_metadata
+      - tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password
+      - tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair
+      - tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name
+      - tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address
+      - tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_reboot_deleted_server
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256
+      - tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name
+      - tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas
+      - tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas
+      - tempest.api.compute.test_versions.TestVersions.test_list_api_versions
+      - tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume
+      - tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments
+      - tempest.api.identity.v3.TestApiDiscovery.test_api_media_types
+      - tempest.api.identity.v3.TestApiDiscovery.test_api_version_resources
+      - tempest.api.identity.v3.TestApiDiscovery.test_api_version_statuses
+      - tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types
+      - tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources
+      - tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses
+      - tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token
+      - tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image
+      - tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image
+      - tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema
+      - tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema
+      - tempest.api.image.v2.test_images.ListImagesTest.test_index_no_params
+      - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format
+      - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format
+      - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_limit
+      - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_min_max_size
+      - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_size
+      - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_status
+      - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_visibility
+      - tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_get_image_schema
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_get_images_schema
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_container_format
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_disk_format
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_limit
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_min_max_size
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_size
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_status
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_visibility
+      - tempest.api.image.v2.test_images.ListUserImagesTest.test_list_no_params
+      - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id
+      - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image
+      - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image
+      - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id
+      - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image
+      - tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image
+      - tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag
+      - tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image
+      - tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes
+      - tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools
+      - tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled
+      - tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw
+      - tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools
+      - tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers
+      - tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway
+      - tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet
+      - tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet
+      - tempest.api.network.test_networks.NetworksTest.test_list_networks
+      - tempest.api.network.test_networks.NetworksTest.test_list_networks_fields
+      - tempest.api.network.test_networks.NetworksTest.test_list_subnets
+      - tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields
+      - tempest.api.network.test_networks.NetworksTest.test_show_network
+      - tempest.api.network.test_networks.NetworksTest.test_show_network_fields
+      - tempest.api.network.test_networks.NetworksTest.test_show_subnet
+      - tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields
+      - tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp
+      - tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_all_attributes
+      - tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_allocation_pools
+      - tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_dhcp_enabled
+      - tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_gw
+      - tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_gw_and_allocation_pools
+      - tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_host_routes_and_dns_nameservers
+      - tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_without_gateway
+      - tempest.api.network.test_networks.NetworksTestJSON.test_create_update_delete_network_subnet
+      - tempest.api.network.test_networks.NetworksTestJSON.test_delete_network_with_subnet
+      - tempest.api.network.test_networks.NetworksTestJSON.test_list_networks
+      - tempest.api.network.test_networks.NetworksTestJSON.test_list_networks_fields
+      - tempest.api.network.test_networks.NetworksTestJSON.test_list_subnets
+      - tempest.api.network.test_networks.NetworksTestJSON.test_list_subnets_fields
+      - tempest.api.network.test_networks.NetworksTestJSON.test_show_network
+      - tempest.api.network.test_networks.NetworksTestJSON.test_show_network_fields
+      - tempest.api.network.test_networks.NetworksTestJSON.test_show_subnet
+      - tempest.api.network.test_networks.NetworksTestJSON.test_show_subnet_fields
+      - tempest.api.network.test_networks.NetworksTestJSON.test_update_subnet_gw_dns_host_routes_dhcp
+      - tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port
+      - tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools
+      - tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port
+      - tempest.api.network.test_ports.PortsTestJSON.test_list_ports
+      - tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields
+      - tempest.api.network.test_ports.PortsTestJSON.test_show_port
+      - tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields
+      - tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes
+      - tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes
+      - tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group
+      - tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args
+      - tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code
+      - tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value
+      - tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id
+      - tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix
+      - tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule
+      - tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group
+      - tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule
+      - tempest.api.volume.test_availability_zone.AvailabilityZoneV2TestJSON.test_get_availability_zone_list
+      - tempest.api.volume.test_extensions.ExtensionsV2TestJSON.test_list_extensions
+      - tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_create_get_delete_snapshot_metadata
+      - tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_crud_snapshot_metadata
+      - tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_update_snapshot_metadata_item
+      - tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_create_get_delete_volume_metadata
+      - tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_crud_volume_metadata
+      - tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item
+      - tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_attach_detach_volume_to_instance
+      - tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_get_volume_attachment
+      - tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_reserve_unreserve_volume
+      - tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_bootable
+      - tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_readonly_update
+      - tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete
+      - tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_as_clone
+      - tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_from_image
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_by_name
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_by_name
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_param_display_name_and_status
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_display_name_and_status
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_metadata
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_details
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_param_metadata
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_availability_zone
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_status
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_availability_zone
+      - tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_status
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_attach_volumes_with_nonexistent_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_invalid_size
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_snapshot_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_source_volid
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_volume_type
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_out_passing_size
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_negative
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_zero
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_without_passing_size
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_invalid_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_volume_without_passing_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_detach_volumes_with_invalid_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_invalid_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_volume_without_passing_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_invalid_status
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_nonexistent_name
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_invalid_status
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_nonexistent_name
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_negative_volume_status
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_nonexistent_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_unreserve_volume_with_nonexistent_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_empty_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_invalid_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_nonexistent_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_delete_nonexistent_volume_id
+      - tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_get_nonexistent_volume_id
+      - tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshot_create_get_list_update_delete
+      - tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_details_with_params
+      - tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_with_params
+      - tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_volume_from_snapshot
+      - tempest.api.volume.test_volumes_snapshots_list.VolumesV2SnapshotListTestJSON.test_snapshots_list_details_with_params
+      - tempest.api.volume.test_volumes_snapshots_list.VolumesV2SnapshotListTestJSON.test_snapshots_list_with_params
+      - tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id
+      - tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id
+      - tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_pagination
+      - tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_with_multiple_params
+      - tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_pagination