summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--docs/testing/developer/testscope/index.rst453
-rw-r--r--docs/testing/user/testspecification/vpn/index.rst487
-rw-r--r--dovetail/compliance/proposed_tests.yml4
-rw-r--r--dovetail/conf/yardstick_config.yml2
-rw-r--r--dovetail/container.py5
-rw-r--r--dovetail/report.py4
-rwxr-xr-xdovetail/run.py39
7 files changed, 727 insertions, 267 deletions
diff --git a/docs/testing/developer/testscope/index.rst b/docs/testing/developer/testscope/index.rst
index ffa91fd1..f9e5ab7d 100644
--- a/docs/testing/developer/testscope/index.rst
+++ b/docs/testing/developer/testscope/index.rst
@@ -6,8 +6,8 @@
Compliance and Verification program accepted test cases
=======================================================
-.. toctree::
- :maxdepth: 2
+ .. toctree::
+ :maxdepth: 2
Mandatory CVP Test Areas
@@ -19,105 +19,112 @@ Test Area VIM Operations - Compute
Image operations within the Compute API
---------------------------------------
-tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image
-tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name
+
+| tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image
+| tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name
Basic support Compute API for server actions such as reboot, rebuild, resize
----------------------------------------------------------------------------
-tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action
-tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions
+
+| tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action
+| tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions
Generate, import, and delete SSH keys within Compute services
-------------------------------------------------------------
-tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair
+
+| tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair
List supported versions of the Compute API
------------------------------------------
-tempest.api.compute.test_versions.TestVersions.test_list_api_versions
+
+| tempest.api.compute.test_versions.TestVersions.test_list_api_versions
Quotas management in Compute API
--------------------------------
-tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas
-tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas
+
+| tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas
+| tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas
Basic server operations in the Compute API
------------------------------------------
-tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password
-tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server
-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name
-tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address
-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers
-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail
-tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers
-tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status
-tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing
-tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server
-tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server
-tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_delete_server_metadata_item
-tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_get_server_metadata_item
-tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_list_server_metadata
-tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata
-tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata_item
-tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_server_metadata
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank
-tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server
-tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server
-tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server
-tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address
-tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256
-tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name
-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus
-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details
-tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus
-tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details
+
+| tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password
+| tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server
+| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name
+| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address
+| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers
+| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail
+| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers
+| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status
+| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing
+| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server
+| tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server
+| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_delete_server_metadata_item
+| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_get_server_metadata_item
+| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_list_server_metadata
+| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata
+| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata_item
+| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_server_metadata
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank
+| tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server
+| tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server
+| tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server
+| tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address
+| tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256
+| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name
+| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus
+| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details
+| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus
+| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details
Retrieve volume information through the Compute API
---------------------------------------------------
-tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume
-tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments
+
+| tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume
+| tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments
@@ -127,15 +134,16 @@ Test Area VIM Operations - Identity
API discovery operations within the Identity v3 API
---------------------------------------------------
-tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types
-tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources
-tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses
+
+| tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types
+| tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources
+| tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses
Auth operations within the Identity API
---------------------------------------
-tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token
+| tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token
--------------------------------
@@ -144,42 +152,47 @@ Test Area VIM Operations - Image
Image deletion tests using the Glance v2 API
--------------------------------------------
-tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image
-tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id
-tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image
-tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag
+
+| tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image
+| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id
+| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image
+| tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag
Image get tests using the Glance v2 API
---------------------------------------
-tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema
-tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema
-tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image
-tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id
-tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image
+
+| tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema
+| tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema
+| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image
+| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id
+| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image
CRUD image operations in Images API v2
--------------------------------------
-tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params
+
+| tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params
Image list tests using the Glance v2 API
----------------------------------------
-tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format
-tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format
-tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_limit
-tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_min_max_size
-tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_size
-tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_status
-tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_visibility
+
+| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format
+| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format
+| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_limit
+| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_min_max_size
+| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_size
+| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_status
+| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_visibility
Image update tests using the Glance v2 API
------------------------------------------
-tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image
-tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image
-tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image
+
+| tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image
+| tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image
+| tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image
----------------------------------
@@ -189,56 +202,57 @@ Test Area VIM Operations - Network
Basic CRUD operations on L2 networks and L2 network ports
---------------------------------------------------------
-tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes
-tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools
-tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled
-tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw
-tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools
-tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers
-tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway
-tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet
-tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet
-tempest.api.network.test_networks.NetworksTest.test_list_networks
-tempest.api.network.test_networks.NetworksTest.test_list_networks_fields
-tempest.api.network.test_networks.NetworksTest.test_list_subnets
-tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields
-tempest.api.network.test_networks.NetworksTest.test_show_network
-tempest.api.network.test_networks.NetworksTest.test_show_network_fields
-tempest.api.network.test_networks.NetworksTest.test_show_subnet
-tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields
-tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp
-tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port
-tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools
-tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port
-tempest.api.network.test_ports.PortsTestJSON.test_list_ports
-tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields
-tempest.api.network.test_ports.PortsTestJSON.test_show_port
-tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields
-tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes
-tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes
+| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes
+| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools
+| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled
+| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw
+| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools
+| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers
+| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway
+| tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet
+| tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet
+| tempest.api.network.test_networks.NetworksTest.test_list_networks
+| tempest.api.network.test_networks.NetworksTest.test_list_networks_fields
+| tempest.api.network.test_networks.NetworksTest.test_list_subnets
+| tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields
+| tempest.api.network.test_networks.NetworksTest.test_show_network
+| tempest.api.network.test_networks.NetworksTest.test_show_network_fields
+| tempest.api.network.test_networks.NetworksTest.test_show_subnet
+| tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields
+| tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp
+| tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port
+| tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools
+| tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port
+| tempest.api.network.test_ports.PortsTestJSON.test_list_ports
+| tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields
+| tempest.api.network.test_ports.PortsTestJSON.test_show_port
+| tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields
+| tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes
+| tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes
Basic CRUD operations on security groups
----------------------------------------
-tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group
-tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args
-tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code
-tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value
-tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id
-tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix
-tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule
-tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group
-tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule
+
+| tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group
+| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args
+| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code
+| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value
+| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id
+| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix
+| tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule
+| tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group
+| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule
---------------------------------
@@ -247,117 +261,142 @@ Test Area VIM Operations - Volume
Volume attach and detach operations with the Cinder v2 API
----------------------------------------------------------
-tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_attach_detach_volume_to_instance
-tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_get_volume_attachment
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_attach_volumes_with_nonexistent_volume_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_detach_volumes_with_invalid_volume_id
+
+| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_attach_detach_volume_to_instance
+| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_get_volume_attachment
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_attach_volumes_with_nonexistent_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_detach_volumes_with_invalid_volume_id
Volume service availability zone operations with the Cinder v2 API
------------------------------------------------------------------
-tempest.api.volume.test_availability_zone.AvailabilityZoneV2TestJSON.test_get_availability_zone_list
+
+| tempest.api.volume.test_availability_zone.AvailabilityZoneV2TestJSON.test_get_availability_zone_list
Volume cloning operations with the Cinder v2 API
------------------------------------------------
-tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_as_clone
+
+| tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_as_clone
Image copy-to-volume operations with the Cinder v2 API
------------------------------------------------------
-tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_bootable
-tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_from_image
+
+| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_bootable
+| tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_from_image
Volume creation and deletion operations with the Cinder v2 API
--------------------------------------------------------------
-tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_invalid_size
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_source_volid
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_volume_type
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_out_passing_size
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_negative
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_zero
+
+| tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_invalid_size
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_source_volid
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_volume_type
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_out_passing_size
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_negative
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_zero
Volume service extension listing operations with the Cinder v2 API
------------------------------------------------------------------
-tempest.api.volume.test_extensions.ExtensionsV2TestJSON.test_list_extensions
+
+| tempest.api.volume.test_extensions.ExtensionsV2TestJSON.test_list_extensions
Volume GET operations with the Cinder v2 API
--------------------------------------------
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_invalid_volume_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_volume_without_passing_volume_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_get_nonexistent_volume_id
+
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_invalid_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_volume_without_passing_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_get_nonexistent_volume_id
+
Volume listing operations with the Cinder v2 API
------------------------------------------------
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_by_name
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_by_name
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_param_display_name_and_status
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_display_name_and_status
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_metadata
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_details
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_param_metadata
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_availability_zone
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_status
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_availability_zone
-tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_status
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_invalid_status
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_nonexistent_name
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_invalid_status
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_nonexistent_name
-tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_pagination
-tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_with_multiple_params
-tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_pagination
+
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_by_name
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_by_name
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_param_display_name_and_status
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_display_name_and_status
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_metadata
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_details
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_param_metadata
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_availability_zone
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_status
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_availability_zone
+| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_status
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_invalid_status
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_nonexistent_name
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_invalid_status
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_nonexistent_name
+| tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_pagination
+| tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_with_multiple_params
+| tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_pagination
Volume metadata operations with the Cinder v2 API
-------------------------------------------------
-tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_create_get_delete_volume_metadata
-tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item
+| tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_create_get_delete_volume_metadata
+| tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item
Verification of read-only status on volumes with the Cinder v2 API
------------------------------------------------------------------
-tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_readonly_update
+
+| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_readonly_update
Volume reservation operations with the Cinder v2 API
----------------------------------------------------
-tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_reserve_unreserve_volume
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_negative_volume_status
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_nonexistent_volume_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_unreserve_volume_with_nonexistent_volume_id
+
+| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_reserve_unreserve_volume
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_negative_volume_status
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_nonexistent_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_unreserve_volume_with_nonexistent_volume_id
Volume snapshot creation/deletion operations with the Cinder v2 API
-------------------------------------------------------------------
-tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_create_get_delete_snapshot_metadata
-tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_update_snapshot_metadata_item
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_snapshot_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_invalid_volume_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_volume_without_passing_volume_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_delete_nonexistent_volume_id
-tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshot_create_get_list_update_delete
-tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_volume_from_snapshot
-tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_details_with_params
-tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_with_params
-tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id
-tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id
+
+| tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_create_get_delete_snapshot_metadata
+| tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_update_snapshot_metadata_item
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_snapshot_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_invalid_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_volume_without_passing_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_delete_nonexistent_volume_id
+| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshot_create_get_list_update_delete
+| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_volume_from_snapshot
+| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_details_with_params
+| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_with_params
+| tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id
+| tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id
Volume update operations with the Cinder v2 API
-----------------------------------------------
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_empty_volume_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_invalid_volume_id
-tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_nonexistent_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_empty_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_invalid_volume_id
+| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_nonexistent_volume_id
Optional CVP Test Areas
========================
+
+-----------------
+Test Area BGP VPN
+-----------------
+
+Verify association and dissasocitation of node using route targets
+------------------------------------------------------------------
+
+| opnfv.sdnvpn.subnet_connectivity
+| opnfv.sdnvpn.tenant separation
+| opnfv.sdnvpn.router_association
+| opnfv.sdnvpn.router_association_floating_ip
+
diff --git a/docs/testing/user/testspecification/vpn/index.rst b/docs/testing/user/testspecification/vpn/index.rst
index 1b5fe439..0a8a8d17 100644
--- a/docs/testing/user/testspecification/vpn/index.rst
+++ b/docs/testing/user/testspecification/vpn/index.rst
@@ -12,14 +12,17 @@ VPN test specification
Scope
=====
-The VPN test area evaluates the ability of the system under test to support VPN networking
-for virtual workdloads. The tests in this suite will evaluate establishing VPN networks,
-publishing and communication between endpoints using BGP and tear down of the networks.
+The VPN test area evaluates the ability of the system under test to support VPN
+networking for virtual workloads. The tests in this test area will evaluate
+establishing VPN networks, publishing and communication between endpoints using
+BGP and tear down of the networks.
References
-================
+==========
-This test suite assumes support for the following specifications:
+This test area evaluates the ability of the system to perform selected actions
+defined in the following specifications. Details of specific features evaluated
+are described in the test descriptions.
- RFC 4364 - BGP/MPLS IP Virtual Private Networks
@@ -33,10 +36,12 @@ This test suite assumes support for the following specifications:
- https://tools.ietf.org/html/rfc2547
+
Definitions and abbreviations
=============================
-The following terms and abreviations are used in conunction with this test suite
+The following terms and abbreviations are used in conjunction with this test
+area
- BGP - Border gateway protocol
- eRT - Export route target
@@ -48,15 +53,27 @@ The following terms and abreviations are used in conunction with this test suite
- VPN - Virtual private network
- VLAN - Virtual local area network
+
System Under Test (SUT)
=======================
-The system under test is assumed to be the NFVi in operation on an Pharos compliant infrastructure.
+The system under test is assumed to be the NFVi and VIM in operation on a
+Pharos compliant infrastructure.
+
-Test Suite Structure
-====================
+Test Area Structure
+===================
+
+The test area is structured in four separate tests which are executed
+sequentially. The order of the tests is arbitrary as there are no dependencies
+across the tests. Specifially, every test performs clean-up operations which
+return the system to the same state as before the test.
+
+The test area evaluates the ability of the SUT to establish connectivity
+between Virtual Machines using an appropriate route target configuration,
+reconfigure the route targets to remove connectivity between the VMs, then
+reestablish connectivity by re-association.
-The test suite is structured in some way that I am unable to articulate at this time.
Test Descriptions
=================
@@ -65,43 +82,451 @@ Test Descriptions
Test Case 1 - VPN provides connectivity between Neutron subnets
----------------------------------------------------------------
+Short name
+----------
+
+opnfv.sdnvpn.subnet_connectivity
+
+
Use case specification
----------------------
-This test evaluate the instance where an NFVi tenant wants to use a BGPVPN to provide
-connectivity between VMs on different Neutron networks and Subnets that reside on different hosts.
+This test evaluates the use case where an NFVi tenant uses a BGPVPN to provide
+connectivity between VMs on different Neutron networks and subnets that reside
+on different hosts.
+
Test preconditions
------------------
-2 compute nodes are available, denoted Node1 and Node 2 in the following.
+2 compute nodes are available, denoted Node1 and Node2 in the following.
+
Basic test flow execution description and pass/fail criteria
------------------------------------------------------------
-Set up VM1 and VM2 on Node1 and VM3 on Node2, all having ports in the same Neutron Network N1
-and all having 10.10.10/24 addresses (this subnet is denoted SN1 in the following).
+Methodology for verifying connectivity
+''''''''''''''''''''''''''''''''''''''
+
+Connectivity between VMs is tested by sending ICMP ping packets between
+selected VMs. The target IPs are passed to the VMs sending pings by means of a
+custom user data script. Whether or not a ping was successful is determined by
+checking the console output of the source VMs.
+
+
+Test execution
+''''''''''''''
+
+* Create Neutron network N1 and subnet SN1 with IP range 10.10.10.0/24
+* Create Neutron network N2 and subnet SN2 with IP range 10.10.11.0/24
+
+* Create VM1 on Node1 with a port in network N1
+* Create VM2 on Node1 with a port in network N1
+* Create VM3 on Node2 with a port in network N1
+* Create VM4 on Node1 with a port in network N2
+* Create VM5 on Node2 with a port in network N2
+
+* Create VPN1 with eRT<>iRT
+* Create network association between network N1 and VPN1
+
+* VM1 sends ICMP packets to VM2 using ``ping``
+
+* **Test assertion 1:** Ping from VM1 to VM2 succeeds: ``ping`` exits with return code 0
+
+* VM1 sends ICMP packets to VM3 using ``ping``
+
+* **Test assertion 2:** Ping from VM1 to VM3 succeeds: ``ping`` exits with return code 0
+
+* VM1 sends ICMP packets to VM4 using ``ping``
+
+* **Test assertion 3:** Ping from VM1 to VM4 fails: ``ping`` exits with a non-zero return code
+
+* Create network association between network N2 and VPN1
+
+* VM4 sends ICMP packets to VM5 using ``ping``
-Set up VM4 on Node1 and VM5 on Node2, both having ports in Neutron Network N2
-and having 10.10.11/24 addresses (this subnet is denoted SN2 in the following).
+* **Test assertion 4:** Ping from VM4 to VM5 succeeds: ``ping`` exits with return code 0
-* Create VPN1 with eRT<>iRT and associate SN1 to it
-* Test action 1: SSH into VM1 and ping VM2, test passes if ping works
-* Test action 2: SSH into VM1 and ping VM3, test passes is ping works
-* Test action 3: SSH into VM1 and ping VM4, test passes if ping does not work
-* Associate SN2 to VPN1
-* Test action 4: Ping from VM4 to VM5 should work
-* Test action 5: Ping from VM1 to VM4 should not work
-* Test action 6: Ping from VM1 to VM5 should not work
* Configure iRT=eRT in VPN1
-* Test action 7: Ping from VM1 to VM4 should work
-* Test action 8: Ping from VM1 to VM5 should work
-The pass criteria for this test case is that all instructions are able to be carried out
-according to the described behaviour without deviation.
-A negative result will be generated if the above is not met in completion.
+* VM1 sends ICMP packets to VM4 using ``ping``
+
+* **Test assertion 5:** Ping from VM1 to VM4 succeeds: ``ping`` exits with return code 0
+
+* VM1 sends ICMP packets to VM5 using ``ping``
+
+* **Test assertion 6:** Ping from VM1 to VM5 succeeds: ``ping`` exits with return code 0
+
+* Delete all instances: VM1, VM2, VM3, VM4 and VM5
+
+* Delete all networks and subnets: networks N1 and N2 including subnets SN1 and SN2
+
+* Delete all network associations and VPN1
+
+
+Pass / fail criteria
+''''''''''''''''''''
+
+This test evaluates the capability of the NFVi and VIM to provide routed IP
+connectivity between VMs by means of BGP/MPLS VPNs. Specifically, the test
+verifies that:
+
+* VMs in the same Neutron subnet have IP connectivity regardless of BGP/MPLS
+ VPNs (test assertion 1, 2, 4)
+
+* VMs in different Neutron subnets do not have IP connectivity by default - in
+ this case without associating VPNs with the same import and export route
+ targets to the Neutron networks (test assertion 3)
+
+* VMs in different Neutron subnets have routed IP connectivity after
+ associating both networks with BGP/MPLS VPNs which have been configured with
+ the same import and export route targets (test assertion 5, 6). Hence,
+ adjusting the ingress and egress route targets enables as well as prohibits
+ routing.
+
+In order to pass this test, all test assertions listed in the test execution
+above need to pass.
+
+
+Post conditions
+---------------
+
+N/A
+
+------------------------------------------------------------
+Test Case 2 - VPNs ensure traffic separation between tenants
+------------------------------------------------------------
+
+Short Name
+----------
+
+opnfv.sdnvpn.tenant_separation
+
+
+Use case specification
+----------------------
+
+This test evaluates if VPNs provide separation of traffic such that overlapping
+IP ranges can be used.
+
+
+Test preconditions
+------------------
+
+2 compute nodes are available, denoted Node1 and Node2 in the following.
+
+
+Basic test flow execution description and pass/fail criteria
+------------------------------------------------------------
+
+Methodology for verifying connectivity
+''''''''''''''''''''''''''''''''''''''
+
+Connectivity between VMs is tested by establishing an SSH connection. Moreover,
+the command "hostname" is executed at the remote VM in order to retrieve the
+hostname of the remote VM. The retrieved hostname is furthermore compared
+against an expected value. This is used to verify tenant traffic separation,
+i.e., despite overlapping IPs, a connection is made to the correct VM as
+determined by means of the hostname of the target VM.
+
+
+
+Test execution
+''''''''''''''
+
+* Create Neutron network N1
+* Create subnet SN1a of network N1 with IP range 10.10.10.0/24
+* Create subnet SN1b of network N1 with IP range 10.10.11.0/24
+
+* Create Neutron network N2
+* Create subnet SN2a of network N2 with IP range 10.10.10.0/24
+* Create subnet SN2b of network N2 with IP range 10.10.11.0/24
+
+* Create VM1 on Node1 with a port in network N1 and IP 10.10.10.11.
+* Create VM2 on Node1 with a port in network N1 and IP 10.10.10.12.
+* Create VM3 on Node2 with a port in network N1 and IP 10.10.11.13.
+* Create VM4 on Node1 with a port in network N2 and IP 10.10.10.12.
+* Create VM5 on Node2 with a port in network N2 and IP 10.10.11.13.
+
+* Create VPN1 with iRT=eRT=RT1
+* Create network association between network N1 and VPN1
+
+* VM1 attempts to execute the command ``hostname`` on the VM with IP 10.10.10.12 via SSH.
+
+* **Test assertion 1:** VM1 can successfully connect to the VM with IP
+ 10.10.10.12. via SSH and execute the remote command ``hostname``. The
+ retrieved hostname equals the hostname of VM2.
+
+* VM1 attempts to execute the command ``hostname`` on the VM with IP 10.10.11.13 via SSH.
+
+* **Test assertion 2:** VM1 can successfully connect to the VM with IP
+ 10.10.11.13 via SSH and execute the remote command ``hostname``. The
+ retrieved hostname equals the hostname of VM3.
+
+* Create VPN2 with iRT=eRT=RT2
+* Create network association between network N2 and VPN2
+
+* VM4 attempts to execute the command ``hostname`` on the VM with IP 10.10.11.13 via SSH.
+
+* **Test assertion 3:** VM4 can successfully connect to the VM with IP
+ 10.10.11.13 via SSH and execute the remote command ``hostname``. The
+ retrieved hostname equals the hostname of VM5.
+
+* VM4 attempts to execute the command ``hostname`` on the VM with IP 10.10.11.11 via SSH.
+
+* **Test assertion 4:** VM4 cannot connect to the VM with IP 10.10.11.11 via SSH.
+
+* Delete all instances: VM1, VM2, VM3, VM4 and VM5
+
+* Delete all networks and subnets: networks N1 and N2 including subnets SN1a, SN1b, SN2a and SN2b
+
+* Delete all network associations, VPN1 and VPN2
+
+
+Pass / fail criteria
+''''''''''''''''''''
+
+This test evaluates the capability of the NFVi and VIM to provide routed IP
+connectivity between VMs by means of BGP/MPLS VPNs. Specifically, the test
+verifies that:
+
+* VMs in the same Neutron subnet (still) have IP connectivity between each
+ other when a BGP/MPLS VPN is associated with the network (test assertion 1).
+
+* VMs in different Neutron subnets have routed IP connectivity between each
+ other when BGP/MPLS VPNs with the same import and expert route targets are
+ associated with both networks (assertion 2).
+
+* VMs in different Neutron networks and BGP/MPLS VPNs with different import and
+ export route targets can have overlapping IP ranges. The BGP/MPLS VPNs
+ provide traffic separation (assertion 3 and 4).
+
+In order to pass this test, all test assertions listed in the test execution
+above need to pass.
+
+
+Post conditions
+---------------
+
+N/A
+
+--------------------------------------------------------------------------------
+Test Case 3 - VPN provides connectivity between subnets using router association
+--------------------------------------------------------------------------------
+
+Short Name
+----------
+
+opnfv.sdnvpn.router_association
+
+
+Use case specification
+----------------------
+
+This test evaluates if a VPN provides connectivity between two subnets by
+utilizing two different VPN association mechanisms: a router association and a
+network association.
+
+Specifically, the test network topology comprises two networks N1 and N2 with
+corresponding subnets. Additionally, network N1 is connected to a router R1.
+This test verifies that a VPN V1 provides connectivity between both networks
+when applying a router association to router R1 and a network association to
+network N2.
+
+
+Test preconditions
+------------------
+
+2 compute nodes are available, denoted Node1 and Node2 in the following.
+
+Basic test flow execution description and pass/fail criteria
+------------------------------------------------------------
+
+Methodology for verifying connectivity
+''''''''''''''''''''''''''''''''''''''
+
+Connectivity between VMs is tested by sending ICMP ping packets between
+selected VMs. The target IPs are passed to the VMs sending pings by means of a
+custom user data script. Whether or not a ping was successful is determined by
+checking the console output of the source VMs.
+
+
+Test execution
+''''''''''''''
+
+* Create a network N1, a subnet SN1 with IP range 10.10.10.0/24 and a connected router R1
+* Create a network N2, a subnet SN2 with IP range 10.10.11.0/24
+
+* Create VM1 on Node1 with a port in network N1
+* Create VM2 on Node1 with a port in network N1
+* Create VM3 on Node2 with a port in network N1
+* Create VM4 on Node1 with a port in network N2
+* Create VM5 on Node2 with a port in network N2
+
+* Create VPN1 with eRT<>iRT so that connected subnets should not reach each other
+
+* Create route association between router R1 and VPN1
+
+* VM1 sends ICMP packets to VM2 using ``ping``
+
+* **Test assertion 1:** Ping from VM1 to VM2 succeeds: ``ping`` exits with return code 0
+
+* VM1 sends ICMP packets to VM3 using ``ping``
+
+* **Test assertion 2:** Ping from VM1 to VM3 succeeds: ``ping`` exits with return code 0
+
+* VM1 sends ICMP packets to VM4 using ``ping``
+
+* **Test assertion 3:** Ping from VM1 to VM4 fails: ``ping`` exits with a non-zero return code
+
+* Create network association between network N2 and VPN1
+
+* VM4 sends ICMP packets to VM5 using ``ping``
+
+* **Test assertion 4:** Ping from VM4 to VM5 succeeds: ``ping`` exits with return code 0
+
+* Change VPN1 so that iRT=eRT
+
+* VM1 sends ICMP packets to VM4 using ``ping``
+
+* **Test assertion 5:** Ping from VM1 to VM4 succeeds: ``ping`` exits with return code 0
+
+* VM1 sends ICMP packets to VM5 using ``ping``
+
+* **Test assertion 6:** Ping from VM1 to VM5 succeeds: ``ping`` exits with return code 0
+
+* Delete all instances: VM1, VM2, VM3, VM4 and VM5
+
+* Delete all networks, subnets and routers: networks N1 and N2 including subnets SN1 and SN2, router R1
+
+* Delete all network and router associations and VPN1
+
+
+Pass / fail criteria
+''''''''''''''''''''
+
+This test evaluates the capability of the NFVi and VIM to provide routed IP
+connectivity between VMs by means of BGP/MPLS VPNs. Specifically, the test
+verifies that:
+
+* VMs in the same Neutron subnet have IP connectivity regardless of the import
+ and export route target configuration of BGP/MPLS VPNs (test assertion 1, 2, 4)
+
+* VMs in different Neutron subnets do not have IP connectivity by default - in
+ this case without associating VPNs with the same import and export route
+ targets to the Neutron networks or connected Neutron routers (test assertion 3).
+
+* VMs in two different Neutron subnets have routed IP connectivity after
+ associating the first network and a router connected to the second network
+ with BGP/MPLS VPNs which have been configured with the same import and export
+ route targets (test assertion 5, 6). Hence, adjusting the ingress and egress
+ route targets enables as well as prohibits routing.
+
+* Network and router associations are equivalent methods for binding Neutron networks
+ to VPN.
+
+In order to pass this test, all test assertions listed in the test execution
+above need to pass.
+
+
+Post conditions
+---------------
+
+N/A
+
+---------------------------------------------------------------------------------------------------
+Test Case 4 - Verify interworking of router and network associations with floating IP functionality
+---------------------------------------------------------------------------------------------------
+
+Short Name
+----------
+
+opnfv.sdnvpn.router_association_floating_ip
+
+
+Use case specification
+----------------------
+
+This test evaluates if both the router association and network association
+mechanisms interwork with floating IP functionality.
+
+Specifically, the test network topology comprises two networks N1 and N2 with
+corresponding subnets. Additionally, network N1 is connected to a router R1.
+This test verifies that i) a VPN V1 provides connectivity between both networks
+when applying a router association to router R1 and a network association to
+network N2 and ii) a VM in network N1 is reachable externally by means of a
+floating IP.
+
+
+Test preconditions
+------------------
+
+At least one compute node is available.
+
+Basic test flow execution description and pass/fail criteria
+------------------------------------------------------------
+
+Methodology for verifying connectivity
+''''''''''''''''''''''''''''''''''''''
+
+Connectivity between VMs is tested by sending ICMP ping packets between
+selected VMs. The target IPs are passed to the VMs sending pings by means of a
+custom user data script. Whether or not a ping was successful is determined by
+checking the console output of the source VMs.
+
+
+Test execution
+''''''''''''''
+
+* Create a network N1, a subnet SN1 with IP range 10.10.10.0/24 and a connected router R1
+* Create a network N2 with IP range 10.10.20.0/24
+
+* Create VM1 with a port in network N1
+* Create VM2 with a port in network N2
+
+* Create VPN1
+* Create a router association between router R1 and VPN1
+* Create a network association between network N2 and VPN1
+
+
+* VM1 sends ICMP packets to VM2 using ``ping``
+
+* **Test assertion 1:** Ping from VM1 to VM2 succeeds: ``ping`` exits with return code 0
+
+* Assign a floating IP to VM1
+
+* The host running the test framework sends ICMP packets to VM1 using ``ping``
+
+* **Test assertion 2:** Ping from the host running the test framework to the
+ floating IP of VM1 succeeds: ``ping`` exits with return code 0
+
+* Delete floating IP assigned to VM1
+
+* Delete all instances: VM1, VM2
+
+* Delete all networks, subnets and routers: networks N1 and N2 including subnets SN1 and SN2, router R1
+
+* Delete all network and router associations as well as VPN1
+
+
+Pass / fail criteria
+''''''''''''''''''''
+
+This test evaluates the capability of the NFVi and VIM to provide routed IP
+connectivity between VMs by means of BGP/MPLS VPNs. Specifically, the test
+verifies that:
+
+* VMs in the same Neutron subnet have IP connectivity regardless of the import
+ and export route target configuration of BGP/MPLS VPNs (test assertion 1)
+
+* VMs connected to a network which has been associated with a BGP/MPLS VPN are
+ reachable through floating IPs.
+
+In order to pass this test, all test assertions listed in the test execution
+above need to pass.
+
Post conditions
---------------
-TBD - should there be any other than the system is in the same state it started out as?
+N/A
diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml
index de892e5d..9d63cb2a 100644
--- a/dovetail/compliance/proposed_tests.yml
+++ b/dovetail/compliance/proposed_tests.yml
@@ -31,9 +31,6 @@ proposed_tests:
- dovetail.ipv6.tc023
- dovetail.ipv6.tc024
- dovetail.ipv6.tc025
- # nfvi, vping_ssh, vping_userdata
- - dovetail.nfvi.tc001
- - dovetail.nfvi.tc002
# HA
- dovetail.ha.tc001
- dovetail.ha.tc002
@@ -46,6 +43,5 @@ proposed_tests:
# sdnvpn
- dovetail.sdnvpn.tc001
- dovetail.sdnvpn.tc002
- - dovetail.sdnvpn.tc003
- dovetail.sdnvpn.tc004
- dovetail.sdnvpn.tc008
diff --git a/dovetail/conf/yardstick_config.yml b/dovetail/conf/yardstick_config.yml
index bc207d7f..9f56cb11 100644
--- a/dovetail/conf/yardstick_config.yml
+++ b/dovetail/conf/yardstick_config.yml
@@ -15,7 +15,7 @@ yardstick:
- 'mkdir -p /home/opnfv/yardstick/results/'
- "cd /home/opnfv/repos/yardstick && source tests/ci/prepare_env.sh &&
yardstick -d task start tests/opnfv/test_cases/{{validate_testcase}}.yaml
- --output-file /home/opnfv/yardstick/results/{{validate_testcase}}.out
+ --output-file /home/opnfv/yardstick/results/{{testcase}}.out
--task-args '{'file': '/home/opnfv/userconfig/pre_config/pod.yaml'}'"
post_condition:
- ''
diff --git a/dovetail/container.py b/dovetail/container.py
index e527a46a..dd73096d 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -102,8 +102,9 @@ class Container(object):
return None
if dovetail_config['report_dest'].startswith("http"):
- cls.logger.info("Yardstick can't push results to DB.")
- cls.logger.info("Results will be stored with files.")
+ envs = envs + " -e DISPATCHER=http"
+ envs = envs + " -e TARGET=%s" % dovetail_config['report_dest']
+ envs = envs + " -e NODE_NAME=master"
log_vol = '-v %s:%s ' % (dovetail_config['result_dir'],
dovetail_config["yardstick"]['result']['log'])
diff --git a/dovetail/report.py b/dovetail/report.py
index 0d83831d..b864de98 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -264,7 +264,7 @@ class FunctestCrawler(object):
"errors": error_case,
"skipped": skipped_case}
except KeyError as e:
- self.logger.error("Key error, exception: %s", e)
+ self.logger.error("Result data don't have key %s.", e)
return None
except ValueError:
continue
@@ -313,7 +313,7 @@ class YardstickCrawler(object):
def crawl_from_file(self, testcase=None):
file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
- testcase.validate_testcase() + '.out')
+ testcase.name() + '.out')
if not os.path.exists(file_path):
self.logger.info('result file not found: %s', file_path)
return None
diff --git a/dovetail/run.py b/dovetail/run.py
index 986ee801..c56b343c 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -77,29 +77,28 @@ def check_tc_result(testcase, logger):
dovetail_result = os.path.join(result_dir,
dt_cfg.dovetail_config['result_file'])
if dt_cfg.dovetail_config['report_dest'].startswith("http"):
- if validate_type.lower() == 'yardstick':
- logger.info("Results have been stored with file %s.",
- os.path.join(result_dir,
- testcase.validate_testcase() + '.out'))
+ if dt_utils.store_db_results(dt_cfg.dovetail_config['report_dest'],
+ dt_cfg.dovetail_config['build_tag'],
+ testcase.name(), dovetail_result,
+ logger):
+ logger.info("Results have been pushed to database and stored "
+ "with local file %s.", dovetail_result)
else:
- if dt_utils.store_db_results(dt_cfg.dovetail_config['report_dest'],
- dt_cfg.dovetail_config['build_tag'],
- testcase.name(), dovetail_result,
- logger):
- logger.info("Results have been pushed to database and stored "
- "with local file %s.", dovetail_result)
- else:
- logger.error("Fail to push results to database.")
+ logger.error("Fail to push results to database.")
if dt_cfg.dovetail_config['report_dest'] == "file":
if validate_type.lower() == 'yardstick':
- logger.info("Results have been stored with file %s.",
- os.path.join(result_dir,
- testcase.validate_testcase() + '.out'))
- if validate_type.lower() == 'functest':
- logger.info("Results have been stored with file %s.",
- os.path.join(result_dir, functest_result))
- result = Report.get_result(testcase)
- Report.check_result(testcase, result)
+ result_file = os.path.join(result_dir, testcase.name() + '.out')
+ elif validate_type.lower() == 'functest':
+ result_file = os.path.join(result_dir, functest_result)
+ else:
+ logger.error("Don't support %s now.", validate_type)
+ return
+ if os.path.isfile(result_file):
+ logger.info("Results have been stored with file %s.", result_file)
+ result = Report.get_result(testcase)
+ Report.check_result(testcase, result)
+ else:
+ logger.error("Fail to store results with file %s.", result_file)
def validate_input(input_dict, check_dict, logger):