diff options
| -rw-r--r-- | docs/testing/developer/testcaserequirements/index.rst | 14 | ||||
| -rw-r--r-- | docs/testing/developer/testscope/index.rst | 363 | ||||
| -rw-r--r-- | docs/testing/user/testspecification/vimoperationscompute/index.rst | 30 | ||||
| -rw-r--r-- | docs/testing/user/testspecification/vpn/index.rst | 107 | ||||
| -rw-r--r-- | dovetail/conf/dovetail_config.yml | 3 | ||||
| -rwxr-xr-x | dovetail/run.py | 9 | ||||
| -rw-r--r-- | dovetail/utils/dovetail_utils.py | 4 |
7 files changed, 508 insertions, 22 deletions
diff --git a/docs/testing/developer/testcaserequirements/index.rst b/docs/testing/developer/testcaserequirements/index.rst index 5dc87713..38eb93a1 100644 --- a/docs/testing/developer/testcaserequirements/index.rst +++ b/docs/testing/developer/testcaserequirements/index.rst @@ -9,17 +9,6 @@ Compliance and Verification program test case requirements .. toctree:: :maxdepth: 2 -Version history -=============== - -+------------+----------+------------------+----------------------------------+ -| **Date** | **Ver.** | **Author** | **Comment** | -| | | | | -+------------+----------+------------------+----------------------------------+ -| 2017-03-15 | 0.0.1 | Chris Price | Draft version | -| | | | | -+------------+----------+------------------+----------------------------------+ - CVP Test Suite Purpose and Goals ================================ @@ -69,12 +58,13 @@ The following requirements are mandatory for test to be submitted for considerat - Use case specification - Test preconditions - Basic test flow execution descriptor - - Post conditions and pass fail criteria + - Pass fail criteria - The following things may be documented for the test case: - Parameter border test cases descriptions - Fault/Error test case descriptions + - Post conditions where the system state may be left changed after completion New test case proposals should complete a CVP test case worksheet to ensure that all of these considerations are met before the test case is approved for inclusion in the diff --git a/docs/testing/developer/testscope/index.rst b/docs/testing/developer/testscope/index.rst new file mode 100644 index 00000000..ffa91fd1 --- /dev/null +++ b/docs/testing/developer/testscope/index.rst @@ -0,0 +1,363 @@ +.. This work is lit_snapshots_list_details_with_paramsensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) Ericsson AB + +======================================================= +Compliance and Verification program accepted test cases +======================================================= + +.. toctree:: + :maxdepth: 2 + + +Mandatory CVP Test Areas +======================== + +---------------------------------- +Test Area VIM Operations - Compute +---------------------------------- + +Image operations within the Compute API +--------------------------------------- +tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image +tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name + + +Basic support Compute API for server actions such as reboot, rebuild, resize +---------------------------------------------------------------------------- +tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action +tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions + + +Generate, import, and delete SSH keys within Compute services +------------------------------------------------------------- +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair + + +List supported versions of the Compute API +------------------------------------------ +tempest.api.compute.test_versions.TestVersions.test_list_api_versions + + +Quotas management in Compute API +-------------------------------- +tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas +tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas + + +Basic server operations in the Compute API +------------------------------------------ +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256 +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_delete_server_metadata_item +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_get_server_metadata_item +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_list_server_metadata +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata_item +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_server_metadata +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server +tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address +tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256 +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details + + +Retrieve volume information through the Compute API +--------------------------------------------------- +tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume +tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments + + + +----------------------------------- +Test Area VIM Operations - Identity +----------------------------------- + +API discovery operations within the Identity v3 API +--------------------------------------------------- +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses + + +Auth operations within the Identity API +--------------------------------------- +tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token + + + +-------------------------------- +Test Area VIM Operations - Image +-------------------------------- + +Image deletion tests using the Glance v2 API +-------------------------------------------- +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image +tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag + + +Image get tests using the Glance v2 API +--------------------------------------- +tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema +tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image + + +CRUD image operations in Images API v2 +-------------------------------------- +tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params + + +Image list tests using the Glance v2 API +---------------------------------------- +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_limit +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_min_max_size +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_size +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_status +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_visibility + + +Image update tests using the Glance v2 API +------------------------------------------ +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image +tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image +tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image + + +---------------------------------- +Test Area VIM Operations - Network +---------------------------------- + +Basic CRUD operations on L2 networks and L2 network ports +--------------------------------------------------------- + +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway +tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet +tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet +tempest.api.network.test_networks.NetworksTest.test_list_networks +tempest.api.network.test_networks.NetworksTest.test_list_networks_fields +tempest.api.network.test_networks.NetworksTest.test_list_subnets +tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields +tempest.api.network.test_networks.NetworksTest.test_show_network +tempest.api.network.test_networks.NetworksTest.test_show_network_fields +tempest.api.network.test_networks.NetworksTest.test_show_subnet +tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields +tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp +tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port +tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools +tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port +tempest.api.network.test_ports.PortsTestJSON.test_list_ports +tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields +tempest.api.network.test_ports.PortsTestJSON.test_show_port +tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields +tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes +tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes + + +Basic CRUD operations on security groups +---------------------------------------- +tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix +tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule +tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule + + +--------------------------------- +Test Area VIM Operations - Volume +--------------------------------- + +Volume attach and detach operations with the Cinder v2 API +---------------------------------------------------------- +tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_attach_detach_volume_to_instance +tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_get_volume_attachment +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_attach_volumes_with_nonexistent_volume_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_detach_volumes_with_invalid_volume_id + + +Volume service availability zone operations with the Cinder v2 API +------------------------------------------------------------------ +tempest.api.volume.test_availability_zone.AvailabilityZoneV2TestJSON.test_get_availability_zone_list + + +Volume cloning operations with the Cinder v2 API +------------------------------------------------ +tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_as_clone + + +Image copy-to-volume operations with the Cinder v2 API +------------------------------------------------------ +tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_bootable +tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_from_image + + +Volume creation and deletion operations with the Cinder v2 API +-------------------------------------------------------------- +tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_invalid_size +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_source_volid +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_volume_type +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_out_passing_size +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_negative +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_zero + + +Volume service extension listing operations with the Cinder v2 API +------------------------------------------------------------------ +tempest.api.volume.test_extensions.ExtensionsV2TestJSON.test_list_extensions + + +Volume GET operations with the Cinder v2 API +-------------------------------------------- +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_invalid_volume_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_volume_without_passing_volume_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_get_nonexistent_volume_id + +Volume listing operations with the Cinder v2 API +------------------------------------------------ +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_by_name +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_by_name +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_param_display_name_and_status +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_display_name_and_status +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_metadata +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_details +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_param_metadata +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_availability_zone +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_status +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_availability_zone +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_status +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_invalid_status +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_nonexistent_name +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_invalid_status +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_nonexistent_name +tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_pagination +tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_with_multiple_params +tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_pagination + + +Volume metadata operations with the Cinder v2 API +------------------------------------------------- + +tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_create_get_delete_volume_metadata +tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item + + +Verification of read-only status on volumes with the Cinder v2 API +------------------------------------------------------------------ +tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_readonly_update + + +Volume reservation operations with the Cinder v2 API +---------------------------------------------------- +tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_reserve_unreserve_volume +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_negative_volume_status +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_reserve_volume_with_nonexistent_volume_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_unreserve_volume_with_nonexistent_volume_id + + +Volume snapshot creation/deletion operations with the Cinder v2 API +------------------------------------------------------------------- +tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_create_get_delete_snapshot_metadata +tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_update_snapshot_metadata_item +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_snapshot_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_invalid_volume_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_volume_without_passing_volume_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_delete_nonexistent_volume_id +tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshot_create_get_list_update_delete +tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_volume_from_snapshot +tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_details_with_params +tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_with_params +tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id +tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id + + +Volume update operations with the Cinder v2 API +----------------------------------------------- +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_empty_volume_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_invalid_volume_id +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_nonexistent_volume_id + + + +Optional CVP Test Areas +======================== + diff --git a/docs/testing/user/testspecification/vimoperationscompute/index.rst b/docs/testing/user/testspecification/vimoperationscompute/index.rst index f8dc5870..4ed37809 100644 --- a/docs/testing/user/testspecification/vimoperationscompute/index.rst +++ b/docs/testing/user/testspecification/vimoperationscompute/index.rst @@ -9,11 +9,31 @@ VIM compute operations test specification .. toctree:: :maxdepth: 2 -Each test case requires documentation according to: -* Use case specification -* Test preconditions -* Basic test flow execution descriptor -* Post conditions and pass fail criteria +Scope +===== + +References +================ + + +Definitions and abbreviations +============================= + + +Use case description +==================== + + +System Under Test (SUT) +======================= + + +Test Suite Structure +==================== + + +Test Descriptions +================= tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers diff --git a/docs/testing/user/testspecification/vpn/index.rst b/docs/testing/user/testspecification/vpn/index.rst new file mode 100644 index 00000000..1b5fe439 --- /dev/null +++ b/docs/testing/user/testspecification/vpn/index.rst @@ -0,0 +1,107 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) Ericsson AB + +====================== +VPN test specification +====================== + +.. toctree:: + :maxdepth: 2 + +Scope +===== + +The VPN test area evaluates the ability of the system under test to support VPN networking +for virtual workdloads. The tests in this suite will evaluate establishing VPN networks, +publishing and communication between endpoints using BGP and tear down of the networks. + +References +================ + +This test suite assumes support for the following specifications: + +- RFC 4364 - BGP/MPLS IP Virtual Private Networks + + - https://tools.ietf.org/html/rfc4364 + +- RFC 4659 - BGP-MPLS IP Virtual Private Network + + - https://tools.ietf.org/html/rfc4659 + +- RFC 2547 - BGP/MPLS VPNs + + - https://tools.ietf.org/html/rfc2547 + +Definitions and abbreviations +============================= + +The following terms and abreviations are used in conunction with this test suite + +- BGP - Border gateway protocol +- eRT - Export route target +- IETF - Internet Engineering Task Force +- iRT - Import route target +- NFVi - Network functions virtualization infrastructure +- Tenant - An isolated set of virtualized infrastructures +- VM - Virtual machine +- VPN - Virtual private network +- VLAN - Virtual local area network + +System Under Test (SUT) +======================= + +The system under test is assumed to be the NFVi in operation on an Pharos compliant infrastructure. + +Test Suite Structure +==================== + +The test suite is structured in some way that I am unable to articulate at this time. + +Test Descriptions +================= + +---------------------------------------------------------------- +Test Case 1 - VPN provides connectivity between Neutron subnets +---------------------------------------------------------------- + +Use case specification +---------------------- + +This test evaluate the instance where an NFVi tenant wants to use a BGPVPN to provide +connectivity between VMs on different Neutron networks and Subnets that reside on different hosts. + +Test preconditions +------------------ + +2 compute nodes are available, denoted Node1 and Node 2 in the following. + +Basic test flow execution description and pass/fail criteria +------------------------------------------------------------ + +Set up VM1 and VM2 on Node1 and VM3 on Node2, all having ports in the same Neutron Network N1 +and all having 10.10.10/24 addresses (this subnet is denoted SN1 in the following). + +Set up VM4 on Node1 and VM5 on Node2, both having ports in Neutron Network N2 +and having 10.10.11/24 addresses (this subnet is denoted SN2 in the following). + +* Create VPN1 with eRT<>iRT and associate SN1 to it +* Test action 1: SSH into VM1 and ping VM2, test passes if ping works +* Test action 2: SSH into VM1 and ping VM3, test passes is ping works +* Test action 3: SSH into VM1 and ping VM4, test passes if ping does not work +* Associate SN2 to VPN1 +* Test action 4: Ping from VM4 to VM5 should work +* Test action 5: Ping from VM1 to VM4 should not work +* Test action 6: Ping from VM1 to VM5 should not work +* Configure iRT=eRT in VPN1 +* Test action 7: Ping from VM1 to VM4 should work +* Test action 8: Ping from VM1 to VM5 should work + +The pass criteria for this test case is that all instructions are able to be carried out +according to the described behaviour without deviation. +A negative result will be generated if the above is not met in completion. + +Post conditions +--------------- + +TBD - should there be any other than the system is in the same state it started out as? diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index 87a4f591..934ff6a6 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -1,7 +1,8 @@ --- -report_file: 'dovetail_report.txt' +# report_file: 'dovetail_report.txt' cli_file_name: 'cmd_config.yml' report_dest: 'file' +result_file: 'results.json' # OPENSTACK Credential file openrc: '/home/opnfv/dovetail/openrc.sh' diff --git a/dovetail/run.py b/dovetail/run.py index 4bea9b53..5b4dca8e 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -74,17 +74,20 @@ def check_tc_result(testcase, logger): result_dir = dt_cfg.dovetail_config['result_dir'] validate_type = testcase.validate_type() functest_result = dt_cfg.dovetail_config['functest']['result']['file_path'] + dovetail_result = os.path.join(result_dir, + dt_cfg.dovetail_config['result_file']) if dt_cfg.dovetail_config['report_dest'].startswith("http"): if validate_type.lower() == 'yardstick': logger.info("Results have been stored with file %s.", os.path.join(result_dir, testcase.validate_testcase() + '.out')) else: - if dt_utils.check_db_results(dt_cfg.dovetail_config['report_dest'], + if dt_utils.store_db_results(dt_cfg.dovetail_config['report_dest'], dt_cfg.dovetail_config['build_tag'], - testcase.name(), + testcase.name(), dovetail_result, logger): - logger.info("Results have been pushed to database.") + logger.info("Results have been pushed to database and stored " + "with local file %s.", dovetail_result) else: logger.error("Fail to push results to database.") if dt_cfg.dovetail_config['report_dest'] == "file": diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py index 83390e9d..e6a775f1 100644 --- a/dovetail/utils/dovetail_utils.py +++ b/dovetail/utils/dovetail_utils.py @@ -132,12 +132,14 @@ def get_ext_net_name(env_file, logger=None): return None -def check_db_results(db_url, build_tag, testcase, logger): +def store_db_results(db_url, build_tag, testcase, dest_file, logger): url = "%s?build_tag=%s-%s" % (db_url, build_tag, testcase) logger.debug("Query to rest api: %s", url) try: data = json.load(urllib2.urlopen(url)) if data['results']: + with open(dest_file, 'a') as f: + f.write(json.dumps(data['results'][0]) + '\n') return True else: return False |