diff options
| -rw-r--r-- | docs/testing/developer/testscope/index.rst | 453 | ||||
| -rw-r--r-- | docs/testing/user/testspecification/vpn/index.rst | 487 | ||||
| -rw-r--r-- | dovetail/compliance/proposed_tests.yml | 4 | ||||
| -rw-r--r-- | dovetail/conf/yardstick_config.yml | 2 | ||||
| -rw-r--r-- | dovetail/container.py | 5 | ||||
| -rw-r--r-- | dovetail/report.py | 4 | ||||
| -rwxr-xr-x | dovetail/run.py | 39 |
7 files changed, 727 insertions, 267 deletions
diff --git a/docs/testing/developer/testscope/index.rst b/docs/testing/developer/testscope/index.rst index ffa91fd1..f9e5ab7d 100644 --- a/docs/testing/developer/testscope/index.rst +++ b/docs/testing/developer/testscope/index.rst @@ -6,8 +6,8 @@ Compliance and Verification program accepted test cases ======================================================= -.. toctree:: - :maxdepth: 2 + .. toctree:: + :maxdepth: 2 Mandatory CVP Test Areas @@ -19,105 +19,112 @@ Test Area VIM Operations - Compute Image operations within the Compute API --------------------------------------- -tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image -tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name + +| tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image +| tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name Basic support Compute API for server actions such as reboot, rebuild, resize ---------------------------------------------------------------------------- -tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action -tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions + +| tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action +| tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions Generate, import, and delete SSH keys within Compute services ------------------------------------------------------------- -tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair + +| tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair List supported versions of the Compute API ------------------------------------------ -tempest.api.compute.test_versions.TestVersions.test_list_api_versions + +| tempest.api.compute.test_versions.TestVersions.test_list_api_versions Quotas management in Compute API -------------------------------- -tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas -tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas + +| tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas +| tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas Basic server operations in the Compute API ------------------------------------------ -tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password -tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256 -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server -tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name -tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address -tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers -tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail -tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers -tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status -tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing -tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server -tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server -tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_delete_server_metadata_item -tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_get_server_metadata_item -tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_list_server_metadata -tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata -tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata_item -tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_server_metadata -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank -tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server -tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server -tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server -tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address -tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256 -tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name -tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus -tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details -tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus -tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details + +| tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password +| tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256 +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server +| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name +| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address +| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers +| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail +| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers +| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status +| tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing +| tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server +| tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server +| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_delete_server_metadata_item +| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_get_server_metadata_item +| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_list_server_metadata +| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata +| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata_item +| tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_server_metadata +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank +| tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server +| tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server +| tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server +| tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address +| tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256 +| tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name +| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus +| tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details +| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus +| tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details Retrieve volume information through the Compute API --------------------------------------------------- -tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume -tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments + +| tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume +| tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments @@ -127,15 +134,16 @@ Test Area VIM Operations - Identity API discovery operations within the Identity v3 API --------------------------------------------------- -tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types -tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources -tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses + +| tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types +| tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources +| tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses Auth operations within the Identity API --------------------------------------- -tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token +| tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token -------------------------------- @@ -144,42 +152,47 @@ Test Area VIM Operations - Image Image deletion tests using the Glance v2 API -------------------------------------------- -tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image -tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id -tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image -tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag + +| tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image +| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id +| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image +| tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag Image get tests using the Glance v2 API --------------------------------------- -tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema -tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema -tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image -tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id -tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image + +| tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema +| tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema +| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image +| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id +| tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image CRUD image operations in Images API v2 -------------------------------------- -tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params + +| tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params Image list tests using the Glance v2 API ---------------------------------------- -tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format -tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format -tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_limit -tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_min_max_size -tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_size -tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_status -tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_visibility + +| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format +| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format +| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_limit +| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_min_max_size +| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_size +| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_status +| tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_visibility Image update tests using the Glance v2 API ------------------------------------------ -tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image -tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image -tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image + +| tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image +| tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image +| tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image ---------------------------------- @@ -189,56 +202,57 @@ Test Area VIM Operations - Network Basic CRUD operations on L2 networks and L2 network ports --------------------------------------------------------- -tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes -tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools -tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled -tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw -tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools -tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers -tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway -tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet -tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet -tempest.api.network.test_networks.NetworksTest.test_list_networks -tempest.api.network.test_networks.NetworksTest.test_list_networks_fields -tempest.api.network.test_networks.NetworksTest.test_list_subnets -tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields -tempest.api.network.test_networks.NetworksTest.test_show_network -tempest.api.network.test_networks.NetworksTest.test_show_network_fields -tempest.api.network.test_networks.NetworksTest.test_show_subnet -tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields -tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp -tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port -tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools -tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port -tempest.api.network.test_ports.PortsTestJSON.test_list_ports -tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields -tempest.api.network.test_ports.PortsTestJSON.test_show_port -tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields -tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes -tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes +| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes +| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools +| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled +| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw +| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools +| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers +| tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway +| tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet +| tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet +| tempest.api.network.test_networks.NetworksTest.test_list_networks +| tempest.api.network.test_networks.NetworksTest.test_list_networks_fields +| tempest.api.network.test_networks.NetworksTest.test_list_subnets +| tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields +| tempest.api.network.test_networks.NetworksTest.test_show_network +| tempest.api.network.test_networks.NetworksTest.test_show_network_fields +| tempest.api.network.test_networks.NetworksTest.test_show_subnet +| tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields +| tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp +| tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port +| tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools +| tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port +| tempest.api.network.test_ports.PortsTestJSON.test_list_ports +| tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields +| tempest.api.network.test_ports.PortsTestJSON.test_show_port +| tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields +| tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes +| tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes Basic CRUD operations on security groups ---------------------------------------- -tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group -tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args -tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code -tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value -tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id -tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix -tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule -tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group -tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule + +| tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group +| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args +| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code +| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value +| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id +| tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix +| tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule +| tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group +| tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule --------------------------------- @@ -247,117 +261,142 @@ Test Area VIM Operations - Volume Volume attach and detach operations with the Cinder v2 API ---------------------------------------------------------- -tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_attach_detach_volume_to_instance -tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_get_volume_attachment -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_attach_volumes_with_nonexistent_volume_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_detach_volumes_with_invalid_volume_id + +| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_attach_detach_volume_to_instance +| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_get_volume_attachment +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_attach_volumes_with_nonexistent_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_detach_volumes_with_invalid_volume_id Volume service availability zone operations with the Cinder v2 API ------------------------------------------------------------------ -tempest.api.volume.test_availability_zone.AvailabilityZoneV2TestJSON.test_get_availability_zone_list + +| tempest.api.volume.test_availability_zone.AvailabilityZoneV2TestJSON.test_get_availability_zone_list Volume cloning operations with the Cinder v2 API ------------------------------------------------ -tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_as_clone + +| tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_as_clone Image copy-to-volume operations with the Cinder v2 API ------------------------------------------------------ -tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_bootable -tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_from_image + +| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_bootable +| tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_from_image Volume creation and deletion operations with the Cinder v2 API -------------------------------------------------------------- -tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_invalid_size -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_source_volid -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_volume_type -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_out_passing_size -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_negative -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_zero + +| tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_invalid_size +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_source_volid +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_volume_type +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_out_passing_size +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_negative +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_zero Volume service extension listing operations with the Cinder v2 API ------------------------------------------------------------------ -tempest.api.volume.test_extensions.ExtensionsV2TestJSON.test_list_extensions + +| tempest.api.volume.test_extensions.ExtensionsV2TestJSON.test_list_extensions Volume GET operations with the Cinder v2 API -------------------------------------------- -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_invalid_volume_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_volume_without_passing_volume_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_get_nonexistent_volume_id + +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_invalid_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_volume_without_passing_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_get_nonexistent_volume_id + Volume listing operations with the Cinder v2 API ------------------------------------------------ -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_by_name -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_by_name -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_param_display_name_and_status -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_display_name_and_status -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_metadata -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_details -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_param_metadata -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_availability_zone -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_status -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_availability_zone -tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_status -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_invalid_status -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_nonexistent_name -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_invalid_status -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_nonexistent_name -tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_pagination -tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_with_multiple_params -tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_pagination + +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_by_name +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_by_name +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_param_display_name_and_status +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_display_name_and_status +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_metadata +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_details +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_param_metadata +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_availability_zone +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_status +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_availability_zone +| tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_status +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_invalid_status +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_nonexistent_name +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_invalid_status +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_nonexistent_name +| tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_pagination +| tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_with_multiple_params +| tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_pagination Volume metadata operations with the Cinder v2 API ------------------------------------------------- -tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_create_get_delete_volume_metadata -tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item +| tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_create_get_delete_volume_metadata +| tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item Verification of read-only status on volumes with the Cinder v2 API ------------------------------------------------------------------ -tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_readonly_update + +| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_readonly_update Volume reservation operations with the Cinder v2 API ---------------------------------------------------- -tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_reserve_unreserve_volume -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_negative_volume_status -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_nonexistent_volume_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_unreserve_volume_with_nonexistent_volume_id + +| tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_reserve_unreserve_volume +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_negative_volume_status +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_nonexistent_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_unreserve_volume_with_nonexistent_volume_id Volume snapshot creation/deletion operations with the Cinder v2 API ------------------------------------------------------------------- -tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_create_get_delete_snapshot_metadata -tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_update_snapshot_metadata_item -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_snapshot_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_invalid_volume_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_volume_without_passing_volume_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_delete_nonexistent_volume_id -tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshot_create_get_list_update_delete -tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_volume_from_snapshot -tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_details_with_params -tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_with_params -tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id -tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id + +| tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_create_get_delete_snapshot_metadata +| tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_update_snapshot_metadata_item +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_snapshot_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_invalid_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_volume_without_passing_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_delete_nonexistent_volume_id +| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshot_create_get_list_update_delete +| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_volume_from_snapshot +| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_details_with_params +| tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_with_params +| tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id +| tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id Volume update operations with the Cinder v2 API ----------------------------------------------- -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_empty_volume_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_invalid_volume_id -tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_nonexistent_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_empty_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_invalid_volume_id +| tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_nonexistent_volume_id Optional CVP Test Areas ======================== + +----------------- +Test Area BGP VPN +----------------- + +Verify association and dissasocitation of node using route targets +------------------------------------------------------------------ + +| opnfv.sdnvpn.subnet_connectivity +| opnfv.sdnvpn.tenant separation +| opnfv.sdnvpn.router_association +| opnfv.sdnvpn.router_association_floating_ip + diff --git a/docs/testing/user/testspecification/vpn/index.rst b/docs/testing/user/testspecification/vpn/index.rst index 1b5fe439..0a8a8d17 100644 --- a/docs/testing/user/testspecification/vpn/index.rst +++ b/docs/testing/user/testspecification/vpn/index.rst @@ -12,14 +12,17 @@ VPN test specification Scope ===== -The VPN test area evaluates the ability of the system under test to support VPN networking -for virtual workdloads. The tests in this suite will evaluate establishing VPN networks, -publishing and communication between endpoints using BGP and tear down of the networks. +The VPN test area evaluates the ability of the system under test to support VPN +networking for virtual workloads. The tests in this test area will evaluate +establishing VPN networks, publishing and communication between endpoints using +BGP and tear down of the networks. References -================ +========== -This test suite assumes support for the following specifications: +This test area evaluates the ability of the system to perform selected actions +defined in the following specifications. Details of specific features evaluated +are described in the test descriptions. - RFC 4364 - BGP/MPLS IP Virtual Private Networks @@ -33,10 +36,12 @@ This test suite assumes support for the following specifications: - https://tools.ietf.org/html/rfc2547 + Definitions and abbreviations ============================= -The following terms and abreviations are used in conunction with this test suite +The following terms and abbreviations are used in conjunction with this test +area - BGP - Border gateway protocol - eRT - Export route target @@ -48,15 +53,27 @@ The following terms and abreviations are used in conunction with this test suite - VPN - Virtual private network - VLAN - Virtual local area network + System Under Test (SUT) ======================= -The system under test is assumed to be the NFVi in operation on an Pharos compliant infrastructure. +The system under test is assumed to be the NFVi and VIM in operation on a +Pharos compliant infrastructure. + -Test Suite Structure -==================== +Test Area Structure +=================== + +The test area is structured in four separate tests which are executed +sequentially. The order of the tests is arbitrary as there are no dependencies +across the tests. Specifially, every test performs clean-up operations which +return the system to the same state as before the test. + +The test area evaluates the ability of the SUT to establish connectivity +between Virtual Machines using an appropriate route target configuration, +reconfigure the route targets to remove connectivity between the VMs, then +reestablish connectivity by re-association. -The test suite is structured in some way that I am unable to articulate at this time. Test Descriptions ================= @@ -65,43 +82,451 @@ Test Descriptions Test Case 1 - VPN provides connectivity between Neutron subnets ---------------------------------------------------------------- +Short name +---------- + +opnfv.sdnvpn.subnet_connectivity + + Use case specification ---------------------- -This test evaluate the instance where an NFVi tenant wants to use a BGPVPN to provide -connectivity between VMs on different Neutron networks and Subnets that reside on different hosts. +This test evaluates the use case where an NFVi tenant uses a BGPVPN to provide +connectivity between VMs on different Neutron networks and subnets that reside +on different hosts. + Test preconditions ------------------ -2 compute nodes are available, denoted Node1 and Node 2 in the following. +2 compute nodes are available, denoted Node1 and Node2 in the following. + Basic test flow execution description and pass/fail criteria ------------------------------------------------------------ -Set up VM1 and VM2 on Node1 and VM3 on Node2, all having ports in the same Neutron Network N1 -and all having 10.10.10/24 addresses (this subnet is denoted SN1 in the following). +Methodology for verifying connectivity +'''''''''''''''''''''''''''''''''''''' + +Connectivity between VMs is tested by sending ICMP ping packets between +selected VMs. The target IPs are passed to the VMs sending pings by means of a +custom user data script. Whether or not a ping was successful is determined by +checking the console output of the source VMs. + + +Test execution +'''''''''''''' + +* Create Neutron network N1 and subnet SN1 with IP range 10.10.10.0/24 +* Create Neutron network N2 and subnet SN2 with IP range 10.10.11.0/24 + +* Create VM1 on Node1 with a port in network N1 +* Create VM2 on Node1 with a port in network N1 +* Create VM3 on Node2 with a port in network N1 +* Create VM4 on Node1 with a port in network N2 +* Create VM5 on Node2 with a port in network N2 + +* Create VPN1 with eRT<>iRT +* Create network association between network N1 and VPN1 + +* VM1 sends ICMP packets to VM2 using ``ping`` + +* **Test assertion 1:** Ping from VM1 to VM2 succeeds: ``ping`` exits with return code 0 + +* VM1 sends ICMP packets to VM3 using ``ping`` + +* **Test assertion 2:** Ping from VM1 to VM3 succeeds: ``ping`` exits with return code 0 + +* VM1 sends ICMP packets to VM4 using ``ping`` + +* **Test assertion 3:** Ping from VM1 to VM4 fails: ``ping`` exits with a non-zero return code + +* Create network association between network N2 and VPN1 + +* VM4 sends ICMP packets to VM5 using ``ping`` -Set up VM4 on Node1 and VM5 on Node2, both having ports in Neutron Network N2 -and having 10.10.11/24 addresses (this subnet is denoted SN2 in the following). +* **Test assertion 4:** Ping from VM4 to VM5 succeeds: ``ping`` exits with return code 0 -* Create VPN1 with eRT<>iRT and associate SN1 to it -* Test action 1: SSH into VM1 and ping VM2, test passes if ping works -* Test action 2: SSH into VM1 and ping VM3, test passes is ping works -* Test action 3: SSH into VM1 and ping VM4, test passes if ping does not work -* Associate SN2 to VPN1 -* Test action 4: Ping from VM4 to VM5 should work -* Test action 5: Ping from VM1 to VM4 should not work -* Test action 6: Ping from VM1 to VM5 should not work * Configure iRT=eRT in VPN1 -* Test action 7: Ping from VM1 to VM4 should work -* Test action 8: Ping from VM1 to VM5 should work -The pass criteria for this test case is that all instructions are able to be carried out -according to the described behaviour without deviation. -A negative result will be generated if the above is not met in completion. +* VM1 sends ICMP packets to VM4 using ``ping`` + +* **Test assertion 5:** Ping from VM1 to VM4 succeeds: ``ping`` exits with return code 0 + +* VM1 sends ICMP packets to VM5 using ``ping`` + +* **Test assertion 6:** Ping from VM1 to VM5 succeeds: ``ping`` exits with return code 0 + +* Delete all instances: VM1, VM2, VM3, VM4 and VM5 + +* Delete all networks and subnets: networks N1 and N2 including subnets SN1 and SN2 + +* Delete all network associations and VPN1 + + +Pass / fail criteria +'''''''''''''''''''' + +This test evaluates the capability of the NFVi and VIM to provide routed IP +connectivity between VMs by means of BGP/MPLS VPNs. Specifically, the test +verifies that: + +* VMs in the same Neutron subnet have IP connectivity regardless of BGP/MPLS + VPNs (test assertion 1, 2, 4) + +* VMs in different Neutron subnets do not have IP connectivity by default - in + this case without associating VPNs with the same import and export route + targets to the Neutron networks (test assertion 3) + +* VMs in different Neutron subnets have routed IP connectivity after + associating both networks with BGP/MPLS VPNs which have been configured with + the same import and export route targets (test assertion 5, 6). Hence, + adjusting the ingress and egress route targets enables as well as prohibits + routing. + +In order to pass this test, all test assertions listed in the test execution +above need to pass. + + +Post conditions +--------------- + +N/A + +------------------------------------------------------------ +Test Case 2 - VPNs ensure traffic separation between tenants +------------------------------------------------------------ + +Short Name +---------- + +opnfv.sdnvpn.tenant_separation + + +Use case specification +---------------------- + +This test evaluates if VPNs provide separation of traffic such that overlapping +IP ranges can be used. + + +Test preconditions +------------------ + +2 compute nodes are available, denoted Node1 and Node2 in the following. + + +Basic test flow execution description and pass/fail criteria +------------------------------------------------------------ + +Methodology for verifying connectivity +'''''''''''''''''''''''''''''''''''''' + +Connectivity between VMs is tested by establishing an SSH connection. Moreover, +the command "hostname" is executed at the remote VM in order to retrieve the +hostname of the remote VM. The retrieved hostname is furthermore compared +against an expected value. This is used to verify tenant traffic separation, +i.e., despite overlapping IPs, a connection is made to the correct VM as +determined by means of the hostname of the target VM. + + + +Test execution +'''''''''''''' + +* Create Neutron network N1 +* Create subnet SN1a of network N1 with IP range 10.10.10.0/24 +* Create subnet SN1b of network N1 with IP range 10.10.11.0/24 + +* Create Neutron network N2 +* Create subnet SN2a of network N2 with IP range 10.10.10.0/24 +* Create subnet SN2b of network N2 with IP range 10.10.11.0/24 + +* Create VM1 on Node1 with a port in network N1 and IP 10.10.10.11. +* Create VM2 on Node1 with a port in network N1 and IP 10.10.10.12. +* Create VM3 on Node2 with a port in network N1 and IP 10.10.11.13. +* Create VM4 on Node1 with a port in network N2 and IP 10.10.10.12. +* Create VM5 on Node2 with a port in network N2 and IP 10.10.11.13. + +* Create VPN1 with iRT=eRT=RT1 +* Create network association between network N1 and VPN1 + +* VM1 attempts to execute the command ``hostname`` on the VM with IP 10.10.10.12 via SSH. + +* **Test assertion 1:** VM1 can successfully connect to the VM with IP + 10.10.10.12. via SSH and execute the remote command ``hostname``. The + retrieved hostname equals the hostname of VM2. + +* VM1 attempts to execute the command ``hostname`` on the VM with IP 10.10.11.13 via SSH. + +* **Test assertion 2:** VM1 can successfully connect to the VM with IP + 10.10.11.13 via SSH and execute the remote command ``hostname``. The + retrieved hostname equals the hostname of VM3. + +* Create VPN2 with iRT=eRT=RT2 +* Create network association between network N2 and VPN2 + +* VM4 attempts to execute the command ``hostname`` on the VM with IP 10.10.11.13 via SSH. + +* **Test assertion 3:** VM4 can successfully connect to the VM with IP + 10.10.11.13 via SSH and execute the remote command ``hostname``. The + retrieved hostname equals the hostname of VM5. + +* VM4 attempts to execute the command ``hostname`` on the VM with IP 10.10.11.11 via SSH. + +* **Test assertion 4:** VM4 cannot connect to the VM with IP 10.10.11.11 via SSH. + +* Delete all instances: VM1, VM2, VM3, VM4 and VM5 + +* Delete all networks and subnets: networks N1 and N2 including subnets SN1a, SN1b, SN2a and SN2b + +* Delete all network associations, VPN1 and VPN2 + + +Pass / fail criteria +'''''''''''''''''''' + +This test evaluates the capability of the NFVi and VIM to provide routed IP +connectivity between VMs by means of BGP/MPLS VPNs. Specifically, the test +verifies that: + +* VMs in the same Neutron subnet (still) have IP connectivity between each + other when a BGP/MPLS VPN is associated with the network (test assertion 1). + +* VMs in different Neutron subnets have routed IP connectivity between each + other when BGP/MPLS VPNs with the same import and expert route targets are + associated with both networks (assertion 2). + +* VMs in different Neutron networks and BGP/MPLS VPNs with different import and + export route targets can have overlapping IP ranges. The BGP/MPLS VPNs + provide traffic separation (assertion 3 and 4). + +In order to pass this test, all test assertions listed in the test execution +above need to pass. + + +Post conditions +--------------- + +N/A + +-------------------------------------------------------------------------------- +Test Case 3 - VPN provides connectivity between subnets using router association +-------------------------------------------------------------------------------- + +Short Name +---------- + +opnfv.sdnvpn.router_association + + +Use case specification +---------------------- + +This test evaluates if a VPN provides connectivity between two subnets by +utilizing two different VPN association mechanisms: a router association and a +network association. + +Specifically, the test network topology comprises two networks N1 and N2 with +corresponding subnets. Additionally, network N1 is connected to a router R1. +This test verifies that a VPN V1 provides connectivity between both networks +when applying a router association to router R1 and a network association to +network N2. + + +Test preconditions +------------------ + +2 compute nodes are available, denoted Node1 and Node2 in the following. + +Basic test flow execution description and pass/fail criteria +------------------------------------------------------------ + +Methodology for verifying connectivity +'''''''''''''''''''''''''''''''''''''' + +Connectivity between VMs is tested by sending ICMP ping packets between +selected VMs. The target IPs are passed to the VMs sending pings by means of a +custom user data script. Whether or not a ping was successful is determined by +checking the console output of the source VMs. + + +Test execution +'''''''''''''' + +* Create a network N1, a subnet SN1 with IP range 10.10.10.0/24 and a connected router R1 +* Create a network N2, a subnet SN2 with IP range 10.10.11.0/24 + +* Create VM1 on Node1 with a port in network N1 +* Create VM2 on Node1 with a port in network N1 +* Create VM3 on Node2 with a port in network N1 +* Create VM4 on Node1 with a port in network N2 +* Create VM5 on Node2 with a port in network N2 + +* Create VPN1 with eRT<>iRT so that connected subnets should not reach each other + +* Create route association between router R1 and VPN1 + +* VM1 sends ICMP packets to VM2 using ``ping`` + +* **Test assertion 1:** Ping from VM1 to VM2 succeeds: ``ping`` exits with return code 0 + +* VM1 sends ICMP packets to VM3 using ``ping`` + +* **Test assertion 2:** Ping from VM1 to VM3 succeeds: ``ping`` exits with return code 0 + +* VM1 sends ICMP packets to VM4 using ``ping`` + +* **Test assertion 3:** Ping from VM1 to VM4 fails: ``ping`` exits with a non-zero return code + +* Create network association between network N2 and VPN1 + +* VM4 sends ICMP packets to VM5 using ``ping`` + +* **Test assertion 4:** Ping from VM4 to VM5 succeeds: ``ping`` exits with return code 0 + +* Change VPN1 so that iRT=eRT + +* VM1 sends ICMP packets to VM4 using ``ping`` + +* **Test assertion 5:** Ping from VM1 to VM4 succeeds: ``ping`` exits with return code 0 + +* VM1 sends ICMP packets to VM5 using ``ping`` + +* **Test assertion 6:** Ping from VM1 to VM5 succeeds: ``ping`` exits with return code 0 + +* Delete all instances: VM1, VM2, VM3, VM4 and VM5 + +* Delete all networks, subnets and routers: networks N1 and N2 including subnets SN1 and SN2, router R1 + +* Delete all network and router associations and VPN1 + + +Pass / fail criteria +'''''''''''''''''''' + +This test evaluates the capability of the NFVi and VIM to provide routed IP +connectivity between VMs by means of BGP/MPLS VPNs. Specifically, the test +verifies that: + +* VMs in the same Neutron subnet have IP connectivity regardless of the import + and export route target configuration of BGP/MPLS VPNs (test assertion 1, 2, 4) + +* VMs in different Neutron subnets do not have IP connectivity by default - in + this case without associating VPNs with the same import and export route + targets to the Neutron networks or connected Neutron routers (test assertion 3). + +* VMs in two different Neutron subnets have routed IP connectivity after + associating the first network and a router connected to the second network + with BGP/MPLS VPNs which have been configured with the same import and export + route targets (test assertion 5, 6). Hence, adjusting the ingress and egress + route targets enables as well as prohibits routing. + +* Network and router associations are equivalent methods for binding Neutron networks + to VPN. + +In order to pass this test, all test assertions listed in the test execution +above need to pass. + + +Post conditions +--------------- + +N/A + +--------------------------------------------------------------------------------------------------- +Test Case 4 - Verify interworking of router and network associations with floating IP functionality +--------------------------------------------------------------------------------------------------- + +Short Name +---------- + +opnfv.sdnvpn.router_association_floating_ip + + +Use case specification +---------------------- + +This test evaluates if both the router association and network association +mechanisms interwork with floating IP functionality. + +Specifically, the test network topology comprises two networks N1 and N2 with +corresponding subnets. Additionally, network N1 is connected to a router R1. +This test verifies that i) a VPN V1 provides connectivity between both networks +when applying a router association to router R1 and a network association to +network N2 and ii) a VM in network N1 is reachable externally by means of a +floating IP. + + +Test preconditions +------------------ + +At least one compute node is available. + +Basic test flow execution description and pass/fail criteria +------------------------------------------------------------ + +Methodology for verifying connectivity +'''''''''''''''''''''''''''''''''''''' + +Connectivity between VMs is tested by sending ICMP ping packets between +selected VMs. The target IPs are passed to the VMs sending pings by means of a +custom user data script. Whether or not a ping was successful is determined by +checking the console output of the source VMs. + + +Test execution +'''''''''''''' + +* Create a network N1, a subnet SN1 with IP range 10.10.10.0/24 and a connected router R1 +* Create a network N2 with IP range 10.10.20.0/24 + +* Create VM1 with a port in network N1 +* Create VM2 with a port in network N2 + +* Create VPN1 +* Create a router association between router R1 and VPN1 +* Create a network association between network N2 and VPN1 + + +* VM1 sends ICMP packets to VM2 using ``ping`` + +* **Test assertion 1:** Ping from VM1 to VM2 succeeds: ``ping`` exits with return code 0 + +* Assign a floating IP to VM1 + +* The host running the test framework sends ICMP packets to VM1 using ``ping`` + +* **Test assertion 2:** Ping from the host running the test framework to the + floating IP of VM1 succeeds: ``ping`` exits with return code 0 + +* Delete floating IP assigned to VM1 + +* Delete all instances: VM1, VM2 + +* Delete all networks, subnets and routers: networks N1 and N2 including subnets SN1 and SN2, router R1 + +* Delete all network and router associations as well as VPN1 + + +Pass / fail criteria +'''''''''''''''''''' + +This test evaluates the capability of the NFVi and VIM to provide routed IP +connectivity between VMs by means of BGP/MPLS VPNs. Specifically, the test +verifies that: + +* VMs in the same Neutron subnet have IP connectivity regardless of the import + and export route target configuration of BGP/MPLS VPNs (test assertion 1) + +* VMs connected to a network which has been associated with a BGP/MPLS VPN are + reachable through floating IPs. + +In order to pass this test, all test assertions listed in the test execution +above need to pass. + Post conditions --------------- -TBD - should there be any other than the system is in the same state it started out as? +N/A diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml index de892e5d..9d63cb2a 100644 --- a/dovetail/compliance/proposed_tests.yml +++ b/dovetail/compliance/proposed_tests.yml @@ -31,9 +31,6 @@ proposed_tests: - dovetail.ipv6.tc023 - dovetail.ipv6.tc024 - dovetail.ipv6.tc025 - # nfvi, vping_ssh, vping_userdata - - dovetail.nfvi.tc001 - - dovetail.nfvi.tc002 # HA - dovetail.ha.tc001 - dovetail.ha.tc002 @@ -46,6 +43,5 @@ proposed_tests: # sdnvpn - dovetail.sdnvpn.tc001 - dovetail.sdnvpn.tc002 - - dovetail.sdnvpn.tc003 - dovetail.sdnvpn.tc004 - dovetail.sdnvpn.tc008 diff --git a/dovetail/conf/yardstick_config.yml b/dovetail/conf/yardstick_config.yml index bc207d7f..9f56cb11 100644 --- a/dovetail/conf/yardstick_config.yml +++ b/dovetail/conf/yardstick_config.yml @@ -15,7 +15,7 @@ yardstick: - 'mkdir -p /home/opnfv/yardstick/results/' - "cd /home/opnfv/repos/yardstick && source tests/ci/prepare_env.sh && yardstick -d task start tests/opnfv/test_cases/{{validate_testcase}}.yaml - --output-file /home/opnfv/yardstick/results/{{validate_testcase}}.out + --output-file /home/opnfv/yardstick/results/{{testcase}}.out --task-args '{'file': '/home/opnfv/userconfig/pre_config/pod.yaml'}'" post_condition: - '' diff --git a/dovetail/container.py b/dovetail/container.py index e527a46a..dd73096d 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -102,8 +102,9 @@ class Container(object): return None if dovetail_config['report_dest'].startswith("http"): - cls.logger.info("Yardstick can't push results to DB.") - cls.logger.info("Results will be stored with files.") + envs = envs + " -e DISPATCHER=http" + envs = envs + " -e TARGET=%s" % dovetail_config['report_dest'] + envs = envs + " -e NODE_NAME=master" log_vol = '-v %s:%s ' % (dovetail_config['result_dir'], dovetail_config["yardstick"]['result']['log']) diff --git a/dovetail/report.py b/dovetail/report.py index 0d83831d..b864de98 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -264,7 +264,7 @@ class FunctestCrawler(object): "errors": error_case, "skipped": skipped_case} except KeyError as e: - self.logger.error("Key error, exception: %s", e) + self.logger.error("Result data don't have key %s.", e) return None except ValueError: continue @@ -313,7 +313,7 @@ class YardstickCrawler(object): def crawl_from_file(self, testcase=None): file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], - testcase.validate_testcase() + '.out') + testcase.name() + '.out') if not os.path.exists(file_path): self.logger.info('result file not found: %s', file_path) return None diff --git a/dovetail/run.py b/dovetail/run.py index 986ee801..c56b343c 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -77,29 +77,28 @@ def check_tc_result(testcase, logger): dovetail_result = os.path.join(result_dir, dt_cfg.dovetail_config['result_file']) if dt_cfg.dovetail_config['report_dest'].startswith("http"): - if validate_type.lower() == 'yardstick': - logger.info("Results have been stored with file %s.", - os.path.join(result_dir, - testcase.validate_testcase() + '.out')) + if dt_utils.store_db_results(dt_cfg.dovetail_config['report_dest'], + dt_cfg.dovetail_config['build_tag'], + testcase.name(), dovetail_result, + logger): + logger.info("Results have been pushed to database and stored " + "with local file %s.", dovetail_result) else: - if dt_utils.store_db_results(dt_cfg.dovetail_config['report_dest'], - dt_cfg.dovetail_config['build_tag'], - testcase.name(), dovetail_result, - logger): - logger.info("Results have been pushed to database and stored " - "with local file %s.", dovetail_result) - else: - logger.error("Fail to push results to database.") + logger.error("Fail to push results to database.") if dt_cfg.dovetail_config['report_dest'] == "file": if validate_type.lower() == 'yardstick': - logger.info("Results have been stored with file %s.", - os.path.join(result_dir, - testcase.validate_testcase() + '.out')) - if validate_type.lower() == 'functest': - logger.info("Results have been stored with file %s.", - os.path.join(result_dir, functest_result)) - result = Report.get_result(testcase) - Report.check_result(testcase, result) + result_file = os.path.join(result_dir, testcase.name() + '.out') + elif validate_type.lower() == 'functest': + result_file = os.path.join(result_dir, functest_result) + else: + logger.error("Don't support %s now.", validate_type) + return + if os.path.isfile(result_file): + logger.info("Results have been stored with file %s.", result_file) + result = Report.get_result(testcase) + Report.check_result(testcase, result) + else: + logger.error("Fail to store results with file %s.", result_file) def validate_input(input_dict, check_dict, logger): |