Merge "Update functest/ci/testcases.yaml"

5 files changed, 334 insertions, 8 deletions

diff --git a/docker/benchmarking-cntt/testcases.yaml b/docker/benchmarking-cntt/testcases.yaml
index c416e83b1..5a29f1346 100644
--- a/docker/benchmarking-cntt/testcases.yaml
+++ b/docker/benchmarking-cntt/testcases.yaml
@@ -1,8 +1,8 @@
 ---
 tiers:
     -
-        name: benchmarking
-        order: 2
+        name: benchmarking_cntt
+        order: 4
         description: >-
             Run several OpenStack performance tools
             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
@@ -66,5 +66,7 @@ tiers:
                     like iperf, iperf3 and netperf (with help of flent). Shaker
                     is able to deploy OpenStack instances and networks in
                     different topologies.
+                dependencies:
+                    - POD_ARCH: '^(?!aarch64$)'
                 run:
                     name: shaker
diff --git a/docker/benchmarking/testcases.yaml b/docker/benchmarking/testcases.yaml
index 45a4a3576..e63481e43 100644
--- a/docker/benchmarking/testcases.yaml
+++ b/docker/benchmarking/testcases.yaml
@@ -2,7 +2,7 @@
 tiers:
     -
         name: benchmarking
-        order: 2
+        order: 3
         description: >-
             Run several OpenStack performance tools
             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
@@ -60,5 +60,7 @@ tiers:
                     like iperf, iperf3 and netperf (with help of flent). Shaker
                     is able to deploy OpenStack instances and networks in
                     different topologies.
+                dependencies:
+                    - POD_ARCH: '^(?!aarch64$)'
                 run:
                     name: shaker
diff --git a/docker/smoke-cntt/testcases.yaml b/docker/smoke-cntt/testcases.yaml
index c15eedf4e..17ce34fe8 100644
--- a/docker/smoke-cntt/testcases.yaml
+++ b/docker/smoke-cntt/testcases.yaml
@@ -1,8 +1,8 @@
 ---
 tiers:
     -
-        name: smoke
-        order: 1
+        name: smoke_cntt
+        order: 2
         description: >-
             Set of basic Functional tests to validate the OPNFV scenarios.
         testcases:
diff --git a/docker/vnf/testcases.yaml b/docker/vnf/testcases.yaml
index 7d5548b38..9822a515a 100644
--- a/docker/vnf/testcases.yaml
+++ b/docker/vnf/testcases.yaml
@@ -2,7 +2,7 @@
 tiers:
     -
         name: vnf
-        order: 4
+        order: 5
         description: >-
             Collection of VNF test cases.
         testcases:
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index e6f3dfbc9..3738ef64f 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -449,9 +449,288 @@ tiers:
                             - '--concurrency=2'
 
     -
-        name: benchmarking
+        name: smoke_cntt
         order: 2
         description: >-
+            Set of basic Functional tests to validate the OPNFV scenarios.
+        testcases:
+            -
+                case_name: tempest_neutron_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 515
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Neutron project. The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                run:
+                    name: tempest_common
+                    args:
+                        mode: "(?!.*admin.test_agent_availability_zone)\
+                            (?!.*admin.test_dhcp_agent_scheduler)\
+                            (?!.*admin.test_l3_agent_scheduler)\
+                            (?!.*admin.test_logging)\
+                            (?!.*admin.test_logging_negative)\
+                            (?!.*admin.test_network_segment_range)\
+                            (?!.*admin.test_ports.PortTestCasesAdmin.test_regenerate_mac_address)\
+                            (?!.*admin.test_ports.PortTestCasesResourceRequest)\
+                            (?!.*admin.test_routers_dvr)\
+                            (?!.*admin.test_routers_flavors)\
+                            (?!.*admin.test_routers_ha)\
+                            (?!.*test_conntrack_helper)\
+                            (?!.*test_floating_ips.FloatingIPPoolTestJSON)\
+                            (?!.*test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_port_details)\
+                            (?!.*test_metering_extensions)\
+                            (?!.*test_metering_negative)\
+                            (?!.*test_networks.NetworksSearchCriteriaTest.test_list_validation_filters)\
+                            (?!.*test_networks.NetworksTestAdmin.test_create_tenant_network_vxlan)\
+                            (?!.*test_networks.NetworksTestJSON.test_create_update_network_dns_domain)\
+                            (?!.*test_port_forwardings)\
+                            (?!.*test_ports.PortsTaggingOnCreation)\
+                            (?!.*test_ports.PortsTestJSON.test_create_port_with_propagate_uplink_status)\
+                            (?!.*test_ports.PortsTestJSON.test_create_port_without_propagate_uplink_status)\
+                            (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_domain)\
+                            (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_name)\
+                            (?!.*test_ports.PortsTestJSON.test_create_update_port_with_no_dns_name)\
+                            (?!.*test_qos.QosMinimumBandwidthRuleTestJSON)\
+                            (?!.*test_revisions.TestRevisions.test_update_dns_domain_bumps_revision)\
+                            (?!.*test_revisions.TestRevisions.test_update_router_extra_attributes_bumps_revision)\
+                            (?!.*test_router_interface_fip)\
+                            (?!.*test_routers.DvrRoutersTest)\
+                            (?!.*test_routers.HaRoutersTest)\
+                            (?!.*test_routers.RoutersIpV6Test.test_extra_routes_atomic)\
+                            (?!.*test_routers.RoutersTest.test_extra_routes_atomic)\
+                            (?!.*test_routers_negative.DvrRoutersNegativeTest)\
+                            (?!.*test_routers_negative.DvrRoutersNegativeTestExtended)\
+                            (?!.*test_routers_negative.HaRoutersNegativeTest)\
+                            (?!.*test_security_groups.RbacSharedSecurityGroupTest)\
+                            (?!.*test_subnetpool_prefix_ops)\
+                            (?!.*test_subnetpools.RbacSubnetPoolTest)\
+                            (?!.*test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_validation_filters)\
+                            (?!.*test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_subnetpool_associate_shared_address_scope)\
+                            (?!.*test_subnets.SubnetsSearchCriteriaTest.test_list_validation_filters)\
+                            (?!.*test_timestamp.TestTimeStamp.test_segment_with_timestamp)\
+                            (?!.*test_trunk.TrunkTestInheritJSONBase.test_add_subport)\
+                            (?!.*test_trunk.TrunkTestMtusJSON)\
+                            (?!.*test_trunk_negative.TrunkTestJSON.test_create_subport_invalid_inherit_network_segmentation_type)\
+                            (?!.*test_trunk_negative.TrunkTestMtusJSON)\
+                            (^neutron_tempest_plugin.api)"
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_cinder_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 8
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Cinder project.
+                run:
+                    name: tempest_common
+                    args:
+                        mode: "(?!.*test_incremental_backup)\
+                            (?!.*test_consistencygroups)\
+                            (^cinder_tempest_plugin.)"
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_keystone_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 27
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Keystone project.
+                run:
+                    name: tempest_common
+                    args:
+                        mode: "(?!.*api.identity.v3.test_oauth1_tokens)\
+                            (?!.*scenario.test_federated_authentication)\
+                            keystone_tempest_plugin."
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_heat_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 120
+                description: >-
+                    This test case runs the Tempest suite proposed by the
+                    Heat project.
+                run:
+                    name: tempest_heat
+                    args:
+                        mode: "(?!.*functional.test_lbaasv2)\
+                          (?!.*RemoteStackTest.test_stack_create_with_cloud_credential)\
+                          (?!.*scenario.test_aodh_alarm)\
+                          (?!.*tests.scenario.test_autoscaling_lb)\
+                          (?!.*scenario.test_autoscaling_lbv2)\
+                          (?!.*scenario.test_server_software_config)\
+                          (?!.*test_volumes.VolumeBackupRestoreIntegrationTest)\
+                          (?!.*scenario.test_octavia_lbaas)\
+                          (?!.*scenario.test_server_cfn_init)\
+                          ^heat_tempest_plugin.tests"
+                        option:
+                            - '--concurrency=1'
+
+            -
+                case_name: rally_sanity_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs a sub group of tests of the OpenStack
+                    Rally suite in smoke mode.
+                run:
+                    name: rally_sanity
+                    args:
+                        tests:
+                            - 'authenticate'
+                            - 'glance'
+                            - 'cinder'
+                            - 'heat'
+                            - 'keystone'
+                            - 'neutron'
+                            - 'nova'
+                            - 'quotas'
+                            - 'swift'
+
+            -
+                case_name: tempest_full_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 1279
+                description: >-
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
+                run:
+                    name: tempest_common
+                    args:
+                        mode: "(?!.*test_fixed_ips)\
+                          (?!.*test_fixed_ips_negative)\
+                          (?!.*test_auto_allocate_network)(?!.*test_floating_ips_bulk)\
+                          (?!.*test_flavors_microversions.FlavorsV255TestJSON)\
+                          (?!.*test_flavors_microversions.FlavorsV261TestJSON)\
+                          (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_iscsi_volume)\
+                          (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_volume_backed_live_migration)\
+                          (?!.*test_live_migration.LiveMigrationTest.test_iscsi_volume)\
+                          (?!.*test_live_migration.LiveMigrationTest.test_volume_backed_live_migration)\
+                          (?!.*test_live_migration.LiveMigrationRemoteConsolesV26Test)\
+                          (?!.*test_quotas.QuotasAdminTestV257)\
+                          (?!.*certificates.test_certificates)\
+                          (?!.*test_quotas_negative.QuotasSecurityGroupAdminNegativeTest)\
+                          (?!.*test_novnc)(?!.*test_server_personality)\
+                          (?!.*test_servers.ServerShowV263Test.test_show_update_rebuild_list_server)\
+                          (?!.*test_servers_microversions.ServerShowV254Test)\
+                          (?!.*test_servers_microversions.ServerShowV257Test)\
+                          (?!.*test_servers_negative.ServersNegativeTestJSON.test_personality_file_contents_not_encoded)\
+                          (?!.*servers.test_virtual_interfaces)\
+                          (?!.*test_server_actions.ServerActionsTestJSON.test_change_server_password)\
+                          (?!.*test_server_actions.ServerActionsTestJSON.test_get_vnc_console)\
+                          (?!.*test_server_actions.ServerActionsTestJSON.test_reboot_server_soft)\
+                          (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
+                          (?!.*test_server_rescue.ServerStableDeviceRescueTest)\
+                          (?!.*test_security_group_default_rules)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_duplicate_name)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_description)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_name)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_des)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_id)\
+                          (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_name)\
+                          (?!.*test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex)\
+                          (?!.*compute.test_virtual_interfaces)(?!.*compute.test_virtual_interfaces_negative)\
+                          (?!.*compute.test_networks)\
+                          (?!.*test_attach_volume.AttachVolumeMultiAttach)\
+                          (?!.*identity.admin.v2)(?!.*identity.v2)\
+                          (?!.*image.v1)\
+                          (?!.*admin.test_dhcp_agent_scheduler)\
+                          (?!.*admin.test_routers_dvr)\
+                          (?!.*test_metering_extensions)(?!.*network.test_tags)\
+                          (?!.*test_routers_negative.DvrRoutersNegativeTest)\
+                          (?!.*test_routers.RoutersIpV6Test.test_create_router_set_gateway_with_fixed_ip)\
+                          (?!.*test_routers.RoutersTest.test_create_router_set_gateway_with_fixed_ip)\
+                          (?!.*test_group_snapshots.GroupSnapshotsV319Test.test_reset_group_snapshot_status)\
+                          (?!.*test_multi_backend)\
+                          (?!.*test_snapshot_manage)\
+                          (?!.*test_volume_retype.VolumeRetypeWithMigrationTest)\
+                          (?!.*test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete)\
+                          (?!.*test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete)\
+                          (?!.*test_volumes_negative.UpdateMultiattachVolumeNegativeTest.test_multiattach_rw_volume_update_failure)\
+                          (?!.*test_volumes_extend.VolumesExtendAttachedTest.test_extend_attached_volume)\
+                          (?!.*\\[.*\\bslow\\b.*\\])(^tempest.api)"
+                        option:
+                            - '--concurrency=4'
+
+            -
+                case_name: tempest_scenario_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 9
+                description: >-
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
+                run:
+                    name: tempest_scenario
+                    args:
+                        mode: "\
+                          (?!.*test_volume_boot_pattern.TestVolumeBootPattern.test_boot_server_from_encrypted_volume_luks)\
+                          (?!.*\\[.*\\bslow\\b.*\\])(^tempest.scenario)"
+                        option:
+                            - '--concurrency=1'
+
+            -
+                case_name: tempest_slow_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                deny_skipping: true
+                tests_count: 43
+                description: >-
+                    The list of test cases is generated by
+                    Tempest automatically and depends on the parameters of
+                    the OpenStack deployment.
+                    https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
+                run:
+                    name: tempest_common
+                    args:
+                        mode: "(?!.*test_volume_swap)\
+                          (?!.*test_server_personality)\
+                          (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
+                          (?!.*test_container_sync.ContainerSyncTest.test_container_synchronization)\
+                          (?!.*test_container_sync_middleware.ContainerSyncMiddlewareTest.test_container_synchronization)\
+                          (?!.*test_encrypted_cinder_volumes)\
+                          (?!.*test_minbw_allocation_placement)\
+                          (?!.*test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling)\
+                          (?!.*test_volume_migrate_attached)\
+                          (?!.*test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert)\
+                          (?=.*\\[.*\\bslow\\b.*\\])(^tempest.)"
+                        option:
+                            - '--concurrency=1'
+
+    -
+        name: benchmarking
+        order: 3
+        description: >-
             Run several OpenStack performance tools
             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
         testcases:
@@ -514,9 +793,52 @@ tiers:
                     name: shaker
 
     -
-        name: vnf
+        name: benchmarking_cntt
         order: 4
         description: >-
+            Run several OpenStack performance tools
+            https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
+        testcases:
+            -
+                case_name: rally_full_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs the full suite of scenarios of the
+                    OpenStack Rally suite using several threads and iterations.
+                run:
+                    name: rally_full
+                    args:
+                        tests:
+                            - 'authenticate'
+                            - 'glance'
+                            - 'cinder'
+                            - 'heat'
+                            - 'keystone'
+                            - 'neutron'
+                            - 'nova'
+                            - 'quotas'
+                            - 'swift'
+
+            -
+                case_name: rally_jobs_cntt
+                project_name: functest
+                criteria: 100
+                blocking: false
+                description: >-
+                    This test case runs a group of Rally jobs used in
+                    OpenStack gating
+                run:
+                    name: rally_jobs
+                    args:
+                        tests:
+                            - 'neutron'
+
+    -
+        name: vnf
+        order: 5
+        description: >-
             Collection of VNF test cases.
         testcases:
             -