From 6f1c45ea2b7193ffb9d89c778d54fee304499bf8 Mon Sep 17 00:00:00 2001 From: morganrOL Date: Thu, 9 Apr 2015 15:47:15 +0200 Subject: Add Rally based bench scenario for OPNFV Add md document to explain how to install Rally and run test manually towards OPNFV solution JIRA:FUNCTEST-1 Change-Id: I8ea1b8907b339ae77b6726ec94b1ff0553cbb1cd Signed-off-by: morganrOL --- .../OpenStack/CI/suites/opnfv-authenticate.json | 107 +++++ .../VIM/OpenStack/CI/suites/opnfv-cinder.json | 325 +++++++++++++++ testcases/VIM/OpenStack/CI/suites/opnfv-heat.json | 238 +++++++++++ .../VIM/OpenStack/CI/suites/opnfv-keystone.json | 76 ++++ .../VIM/OpenStack/CI/suites/opnfv-neutron.json | 336 +++++++++++++++ testcases/VIM/OpenStack/CI/suites/opnfv-nova.json | 452 +++++++++++++++++++++ .../VIM/OpenStack/CI/suites/opnfv-quotas.json | 93 +++++ .../VIM/OpenStack/CI/suites/opnfv-tempest.json | 13 + testcases/VIM/OpenStack/CI/suites/opnfv-vm.json | 66 +++ testcases/VIM/OpenStack/OpenStack.md | 211 ++++++++++ 10 files changed, 1917 insertions(+) create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-heat.json create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-nova.json create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json create mode 100644 testcases/VIM/OpenStack/CI/suites/opnfv-vm.json create mode 100644 testcases/VIM/OpenStack/OpenStack.md (limited to 'testcases/VIM/OpenStack') diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json b/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json new file mode 100644 index 000000000..3ded19963 --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json @@ -0,0 +1,107 @@ +{ + "Authenticate.keystone": [ + { + "runner": { + "type": "constant", + "times": 100, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 50 + } + } + } + ], + "Authenticate.validate_cinder": [ + { + "args": { + "repetitions": 2 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 5 + } + } + } + ], + "Authenticate.validate_glance": [ + { + "args": { + "repetitions": 2 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 5 + } + } + } + ], + "Authenticate.validate_heat": [ + { + "args": { + "repetitions": 2 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 5 + } + } + } + ], + "Authenticate.validate_neutron": [ + { + "args": { + "repetitions": 2 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 5 + } + } + } + ], + "Authenticate.validate_nova": [ + { + "args": { + "repetitions": 2 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 5 + } + } + } + ] +} diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json b/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json new file mode 100644 index 000000000..b65dd3b2c --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json @@ -0,0 +1,325 @@ +{ + "CinderVolumes.create_and_attach_volume": [ + { + "args": { + "size": 10, + "image": { + "name": "^Ubuntu*" + }, + "flavor": { + "name": "m1.tiny" + } + }, + "runner": { + "type": "constant", + "times": 5, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 2 + } + } + } + ], + "CinderVolumes.create_and_delete_snapshot": [ + { + "args": { + "force": false + }, + "runner": { + "type": "constant", + "times": 3, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 2 + }, + "volumes": { + "size": 1 + } + } + } + ], + "CinderVolumes.create_and_delete_volume": [ + { + "args": { + "size": 1 + }, + "runner": { + "type": "constant", + "times": 3, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 2 + } + } + } + ], + "CinderVolumes.create_and_extend_volume": [ + { + "args": { + "size": 1, + "new_size": 2 + }, + "runner": { + "type": "constant", + "times": 2, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + } + } + ], + "CinderVolumes.create_and_list_snapshots": [ + { + "args": { + "force": false, + "detailed": true + }, + "runner": { + "type": "constant", + "times": 2, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "volumes": { + "size": 1 + } + } + } + ], + "CinderVolumes.create_and_list_volume": [ + { + "args": { + "size": 1, + "detailed": true + }, + "runner": { + "type": "constant", + "times": 3, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + } + } + ], + "CinderVolumes.create_and_upload_volume_to_image": [ + { + "args": { + "size": 1, + "force": false, + "container_format": "bare", + "disk_format": "raw", + "do_delete": true + }, + "runner": { + "type": "constant", + "times": 3, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 2 + } + } + } + ], + "CinderVolumes.create_and_delete_volume": [ + { + "args": { + "size": 1, + "image": { + "name": "^Ubuntu*" + } + }, + "runner": { + "type": "constant", + "times": 2, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 2 + } + } + } + ], + "CinderVolumes.create_from_volume_and_delete_volume": [ + { + "args": { + "size": 1 + }, + "runner": { + "type": "constant", + "times": 2, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "volumes": { + "size": 1 + } + } + } + ], + "CinderVolumes.create_nested_snapshots_and_attach_volume": [ + { + "args": { + "size": { + "min": 1, + "max": 5 + }, + "nested_level": { + "min": 5, + "max": 10 + } + }, + "runner": { + "type": "constant", + "times": 1, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 1 + }, + "servers": { + "image": { + "name": "^Ubuntu*" + }, + "flavor": { + "name": "m1.tiny" + }, + "servers_per_tenant": 2 + } + } + } + ], + "CinderVolumes.create_snapshot_and_attach_volume": [ + { + "args": { + "volume_type": false, + "size": { + "min": 1, + "max": 5 + } + }, + "runner": { + "type": "constant", + "times": 4, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 1 + }, + "servers": { + "image": { + "name": "^Ubuntu*" + }, + "flavor": { + "name": "m1.tiny" + }, + "servers_per_tenant": 2 + } + } + }, + { + "args": { + "volume_type": true, + "size": { + "min": 1, + "max": 5 + } + }, + "runner": { + "type": "constant", + "times": 4, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 1 + }, + "servers": { + "image": { + "name": "^Ubuntu*" + }, + "flavor": { + "name": "m1.tiny" + }, + "servers_per_tenant": 2 + } + } + } + + ], + "CinderVolumes.create_volume": [ + { + "args": { + "size": 1 + }, + "runner": { + "type": "constant", + "times": 3, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 2 + } + } + } + ], + "CinderVolumes.list_volumes": [ + { + "args": { + "detailed": true + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "volumes": { + "size": 1, + "volumes_per_tenant": 4 + } + } + } + ] +} + diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json b/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json new file mode 100644 index 000000000..2fbb70fba --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json @@ -0,0 +1,238 @@ +{ + "HeatStacks.create_and_delete_stack": [ + { + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_and_delete_stack": [ + { + "args": { + "template_path": "templates/server_with_ports.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_and_delete_stack": [ + { + "args": { + "template_path": "templates/server_with_volume.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_and_list_stack": [ + { + "runner": { + "type": "constant", + "times": 10, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + } + } + ], + "HeatStacks.create_check_delete_stack": [ + { + "args": { + "template_path": "templates/random_strings.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_suspend_resume_delete_stack": [ + { + "args": { + "template_path": "templates/random_strings.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "HeatStacks.create_update_delete_stack": [ + { + "args": { + "template_path": "templates/random_strings.yaml.template", + "updated_template_path": "templates/updated_random_strings_add.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_update_delete_stack": [ + { + "args": { + "template_path": "templates/random_strings.yaml.template", + "updated_template_path": "templates/updated_random_strings_delete.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_update_delete_stack": [ + { + "args": { + "template_path": "templates/resource_group.yaml.template", + "updated_template_path": "templates/updated_resource_group_increase.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_update_delete_stack": [ + { + "args": { + "template_path": "templates/autoscaling_policy.yaml.template", + "updated_template_path": "templates/updated_autoscaling_policy_inplace.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_update_delete_stack": [ + { + "args": { + "template_path": "templates/resource_group.yaml.template", + "updated_template_path": "templates/updated_resource_group_reduce.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.create_update_delete_stack": [ + { + "args": { + "template_path": "templates/random_strings.yaml.template", + "updated_template_path": "templates/updated_random_strings_replace.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ], + "HeatStacks.list_stacks_and_resources": [ + { + "runner": { + "type": "constant", + "times": 10, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "stacks": { + "stacks_per_tenant": 2, + "resources_per_stack": 10 + } + } + } + ] +} + diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json b/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json new file mode 100644 index 000000000..390a1ae13 --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json @@ -0,0 +1,76 @@ +{ + "KeystoneBasic.create_delete_user": [ + { + "args": { + "name_length": 10 + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + } + } + ], + "KeystoneBasic.create_and_list_tenants": [ + { + "args": { + "name_length": 10 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 1 + } + } + ], + "KeystoneBasic.create_and_list_users": [ + { + "args": { + "name_length": 10 + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + } + } + ], + "KeystoneBasic.create_tenant": [ + { + "args": { + "name_length": 10 + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + } + } + ], + "KeystoneBasic.create_user": [ + { + "args": { + "name_length": 10 + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + } + } + ], + "KeystoneBasic.create_tenant_with_users": [ + { + "args": { + "name_length": 10, + "users_per_tenant": 10 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 10 + } + } + ] +} + diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json b/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json new file mode 100644 index 000000000..5d176ca0d --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json @@ -0,0 +1,336 @@ +{ + "NeutronNetworks.create_and_delete_networks": [ + { + "args": { + "network_create_args": {} + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_delete_ports": [ + { + "args": { + "network_create_args": {}, + "port_create_args": {}, + "ports_per_network": 10 + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1, + "port": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_delete_routers": [ + { + "args": { + "network_create_args": {}, + "subnet_create_args": {}, + "subnet_cidr_start": "1.1.0.0/30", + "subnets_per_network": 2, + "router_create_args": {} + }, + "runner": { + "type": "constant", + "times": 30, + "concurrency": 10 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1, + "subnet": -1, + "router": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_delete_subnets": [ + { + "args": { + "network_create_args": {}, + "subnet_create_args": {}, + "subnet_cidr_start": "1.1.0.0/30", + "subnets_per_network": 2 + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1, + "subnet": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_list_networks": [ + { + "args": { + "network_create_args": {} + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_list_ports": [ + { + "args": { + "network_create_args": {}, + "port_create_args": {}, + "ports_per_network": 10 + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1, + "port": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_list_routers": [ + { + "args": { + "network_create_args": {}, + "subnet_create_args": {}, + "subnet_cidr_start": "1.1.0.0/30", + "subnets_per_network": 2, + "router_create_args": {} + }, + "runner": { + "type": "constant", + "times": 100, + "concurrency": 10 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1, + "subnet": -1, + "router": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_list_subnets": [ + { + "args": { + "network_create_args": {}, + "subnet_create_args": {}, + "subnet_cidr_start": "1.1.0.0/30", + "subnets_per_network": 2 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 5, + "users_per_tenant": 5 + }, + "quotas": { + "neutron": { + "network": -1, + "subnet": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_update_networks": [ + { + "args": { + "network_update_args": { + "admin_state_up": false, + "name": "_updated" + }, + "network_create_args": {} + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_update_ports": [ + { + "args": { + "network_create_args": {}, + "port_create_args": {}, + "port_update_args": { + "admin_state_up": false, + "device_id": "dummy_id", + "device_owner": "dummy_owner", + "name": "_port_updated" + }, + "ports_per_network": 5 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1, + "port": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_update_routers": [ + { + "args": { + "network_create_args": {}, + "subnet_create_args": {}, + "subnet_cidr_start": "1.1.0.0/30", + "subnets_per_network": 2, + "router_create_args": {}, + "router_update_args": { + "admin_state_up": false, + "name": "_router_updated" + } + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "quotas": { + "neutron": { + "network": -1, + "subnet": -1, + "router": -1 + } + } + } + } + ], + "NeutronNetworks.create_and_update_subnets": [ + { + "args": { + "subnet_update_args": { + "enable_dhcp": false, + "name": "_subnet_updated" + }, + "network_create_args": {}, + "subnet_create_args": {}, + "subnet_cidr_start": "1.4.0.0/16", + "subnets_per_network": 2 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 5, + "users_per_tenant": 5 + }, + "quotas": { + "neutron": { + "network": -1, + "subnet": -1 + } + } + } + } + ] +} + diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json b/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json new file mode 100644 index 000000000..74ea15f3b --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json @@ -0,0 +1,452 @@ +{ + "NovaServers.boot_and_delete_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaKeypair.boot_and_delete_server_with_keypair": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + } + }, + "runner": { + "type": "constant", + "times": 5, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 1 + }, + "network": { + "start_cidr": "100.1.0.0/26" + } + } + } + ], + "NovaSecGroup.boot_and_delete_server_with_secgroups": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "security_group_count": 10, + "rules_per_security_group": 10 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + }, + "network": { + "start_cidr": "100.1.0.0/26" + } + } + } + ], + "NovaServers.boot_and_list_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "detailed": true + }, + "runner": { + "type": "constant", + "times": 1, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + } + } + ], + "NovaServers.boot_and_live_migrate_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "block_migration": false + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + } + } + ], + "NovaServers.boot_and_migrate_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + } + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + } + } + ], + "NovaServers.boot_and_bounce_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "force_delete": false, + "actions": [ + {"hard_reboot": 1}, + {"soft_reboot": 1}, + {"stop_start": 1}, + {"rescue_unrescue": 1} + ] + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaServers.boot_server_from_volume_and_delete": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "volume_size": 10, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaServers.boot_server_from_volume": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "volume_size": 10 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaServers.boot_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + } + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaServers.boot_server_attach_created_volume_and_live_migrate": [ + { + "args": { + "size": 10, + "block_migration": false, + "image": { + "name": "^Ubuntu*" + }, + "flavor": { + "name": "m1.small" + } + }, + "runner": { + "type": "constant", + "times": 5, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 2 + } + } + } + ], + "NovaServers.boot_server_from_volume_and_live_migrate": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "block_migration": false, + "volume_size": 10, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + } + } + ], + "NovaServers.snapshot_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaKeypair.create_and_delete_keypair": [ + { + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaSecGroup.create_and_delete_secgroups": [ + { + "args": { + "security_group_count": 10, + "rules_per_security_group": 10 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaKeypair.create_and_list_keypairs": [ + { + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaSecGroup.create_and_list_secgroups": [ + { + "args": { + "security_group_count": 10, + "rules_per_security_group": 10 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "NovaServers.list_servers": [ + { + "args": { + "detailed": true + }, + "runner": { + "type": "constant", + "times": 1, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "servers": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "servers_per_tenant": 2 + } + } + } + ], + "NovaServers.resize_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "to_flavor": { + "name": "m1.small" + }, + "confirm": true, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 5 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + } + } + } + ] +} + diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json b/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json new file mode 100644 index 000000000..1778a8dd0 --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json @@ -0,0 +1,93 @@ +{ + "Quotas.cinder_update_and_delete": [ + { + "args": { + "max_quota": 1024 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "Quotas.cinder_update": [ + { + "args": { + "max_quota": 1024 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "Quotas.neutron_update": [ + { + "args": { + "max_quota": 1024 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "Quotas.nova_update_and_delete": [ + { + "args": { + "max_quota": 1024 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ], + "Quotas.nova_update": [ + { + "args": { + "max_quota": 1024 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ] +} + diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json b/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json new file mode 100644 index 000000000..0b63070a0 --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json @@ -0,0 +1,13 @@ +{ + "TempestScenario.all": [ + { + "args": {"tempest_conf": "/etc/tempest/tempest.conf"}, + "runner": { + "type": "constant", + "times": 1, + "concurrency": 1 + } + } + ] +} + diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json b/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json new file mode 100644 index 000000000..1524d5bf6 --- /dev/null +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json @@ -0,0 +1,66 @@ +{ + "VMTasks.boot_runcommand_delete": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "floating_network": "public", + "force_delete": false, + "script": "samples/tasks/support/instance_dd_test.sh", + "interpreter": "/bin/sh", + "username": "ubuntu" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + }, + "network": { + } + } + } + ], + "VMTasks.boot_runcommand_delete": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^Ubuntu*" + }, + "volume_args": { + "size": 2 + }, + "fixed_network": "private", + "floating_network": "public", + "use_floatingip": true, + "force_delete": false, + "script": "samples/tasks/support/instance_dd_test.sh", + "interpreter": "/bin/sh", + "username": "ubuntu" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ] +} + diff --git a/testcases/VIM/OpenStack/OpenStack.md b/testcases/VIM/OpenStack/OpenStack.md new file mode 100644 index 000000000..a57699312 --- /dev/null +++ b/testcases/VIM/OpenStack/OpenStack.md @@ -0,0 +1,211 @@ +# Rally tests for OpenStack + +Original Rally testsuites can be found here: https://github.com/stackforge/rally + +--- +## Intro +In order to perform functional and performance testing, we use a dedicated VM. + + TODO: add test-tool VM in architecture and puppetise test-tool VM + +On this VM, we installed: +* Rally: https://wiki.openstack.org/wiki/Rally + + TODO RobotFramework: http://robotframework.org, SIPP http://sipp.sourceforge.net/ + + + +## Installation & Configuration + +### Rally + +* Log on the test-tool VM +* install Rally (https://rally.readthedocs.org/en/latest/tutorial/step_0_installation.html) + +openrc file (info from OpenStack) is needed, password is required during configuration procedure + +* check +```bash +# rally deployment check +keystone endpoints are valid and following service are available: ++-------------+-----------+------------+ +| Services | Type | Status | ++-----------+-------------+------------+ +| cinder | volume | Available | +| cinderv2 | volumev2 | Available | +| glance | image | Available | +| keystone | identity | Available | +| neutron | network | Available | +| nova | compute | Available | +| nova_ec2 | compute_ec2 | Available | +| novav3 | computev3 | Available | ++-----------+-------------+------------+ + +``` + +* For Rally scenario, follow https://rally.readthedocs.org/en/latest/tutorial/step_1_setting_up_env_and_running_benchmark_from_samples.html +```bash +# rally task start ./samples/tasks/scenarios/nova/my-boot-and-delete.json +-------------------------------------------------------------------------------- + Preparing input task +-------------------------------------------------------------------------------- + +Input task is: +{ + "NovaServers.boot_and_delete_server": [ + { + "args": { + "flavor": { + "name": "m1.small" + }, + "image": { + "name": "^ubuntu-14.10-64b" + }, + "force_delete": false + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 3, + "users_per_tenant": 2 + } + } + } + ] +} + +-------------------------------------------------------------------------------- + Task f42c8aed-00a6-4715-9951-945b4fb97c32: started +-------------------------------------------------------------------------------- + +Benchmarking... This can take a while... + +To track task status use: + + rally task status + or + rally task detailed + +-------------------------------------------------------------------------------- +Task f42c8aed-00a6-4715-9951-945b4fb97c32: finished +-------------------------------------------------------------------------------- + +test scenario NovaServers.boot_and_delete_server +args position 0 +args values: +OrderedDict([(u'runner', OrderedDict([(u'type', u'constant'), (u'concurrency', 2), (u'times', 10)])), (u'args', OrderedDict([(u'force_delete', False), (u'flavor', OrderedDict([(u'name', u'm1.small')])), (u'image', OrderedDict([(u'name', u'^ubuntu-14.10-64b')]))])), (u'context', OrderedDict([(u'users', OrderedDict([(u'project_domain', u'default'), (u'users_per_tenant', 2), (u'tenants', 3), (u'resource_management_workers', 30), (u'user_domain', u'default')]))]))]) ++--------------------+-----------+-----------+-----------+---------------+---------------+---------+-------+ +| action | min (sec) | avg (sec) | max (sec) | 90 percentile | 95 percentile | success | count | ++--------------------+-----------+-----------+-----------+---------------+---------------+---------+-------+ +| nova.boot_server | 4.675 | 5.554 | 6.357 | 6.289 | 6.323 | 100.0% | 10 | +| nova.delete_server | 2.365 | 3.301 | 4.728 | 4.553 | 4.64 | 100.0% | 10 | +| total | 7.303 | 8.857 | 10.789 | 10.543 | 10.666 | 100.0% | 10 | ++--------------------+-----------+-----------+-----------+---------------+---------------+---------+-------+ +Load duration: 45.7972288132 +Full duration: 58.912060976 + +HINTS: +* To plot HTML graphics with this data, run: + rally task report f42c8aed-00a6-4715-9951-945b4fb97c32 --out output.html + +* To get raw JSON output of task results, run: + rally task results f42c8aed-00a6-4715-9951-945b4fb97c32 + +Using task: f42c8aed-00a6-4715-9951-945b4fb97c32 + +``` +* For Tempest, follow the instructions https://www.mirantis.com/blog/rally-openstack-tempest-testing-made-simpler +* In first step Rally scenario were fine but Tempest scenarios failed due to configuration +Apply patch https://review.openstack.org/#/c/163330/ +```bash +pip uninstall rally && cd ./rally && python setup.py install +``` + +You shall be able to run Rally/Tempest towards your OpenStack +```bash +root@rally:~/rally# rally verify start +[...] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest + test_attach_volumes_with_nonexistent_volume_id[compute,gate,id-f5e56b0a-5d02-43c1-a2a7-c9b792c2e3f6,negative]FAIL + test_create_volume_with_invalid_size[gate,id-1ed83a8a-682d-4dfb-a30e-ee63ffd6c049,negative]OK 0.02 + test_create_volume_with_nonexistent_snapshot_id[gate,id-0c36f6ae-4604-4017-b0a9-34fdc63096f9,negative]OK 0.04 + test_create_volume_with_nonexistent_source_volid[gate,id-47c73e08-4be8-45bb-bfdf-0c4e79b88344,negative]OK 0.05 + test_create_volume_with_nonexistent_volume_type[gate,id-10254ed8-3849-454e-862e-3ab8e6aa01d2,negative]OK 0.02 + test_create_volume_with_out_passing_size[gate,id-9387686f-334f-4d31-a439-33494b9e2683,negative]OK 0.02 + test_create_volume_with_size_negative[gate,id-8b472729-9eba-446e-a83b-916bdb34bef7,negative]OK 0.02 +[...] +Ran 933 tests in 1020.200s + +FAILED (failures=186) +Test set 'full' has been finished with error. Check log for details + +``` + +It is possible to get a better view on the result +```bash +# rally verify list ++--------------------------------------+--------------------------------------+----------+-------+----------+----------------------------+----------------+----------+ +| UUID | Deployment UUID | Set name | Tests | Failures | Created at | Duration | Status | ++--------------------------------------+--------------------------------------+----------+-------+----------+----------------------------+----------------+----------+ +| b1de3608-dbee-40e7-84c4-1c756ca0347c | e7d70ddf-9be0-4681-9456-aa8dce515e0e | None | 0 | 0 | 2015-03-11 08:48:04.416793 | 0:00:00.102275 | running | +| ff0d9285-184f-47d5-9474-7475135ae8cf | e7d70ddf-9be0-4681-9456-aa8dce515e0e | full | 933 | 186 | 2015-03-11 09:57:01.836611 | 0:18:08.360204 | finished | +| fec2fd0a-a4ef-4064-a292-95e9da68025c | e7d70ddf-9be0-4681-9456-aa8dce515e0e | full | 933 | 186 | 2015-03-12 09:46:40.818691 | 0:17:02.316443 | finished | ++--------------------------------------+--------------------------------------+----------+-------+----------+----------------------------+----------------+----------+ + +rally verify show fec2fd0a-a4ef-4064-a292-95e9da68025c +Total results of verification: + ++--------------------------------------+--------------------------------------+----------+-------+----------+----------------------------+----------+ +| UUID | Deployment UUID | Set name | Tests | Failures | Created at | Status | ++--------------------------------------+--------------------------------------+----------+-------+----------+----------------------------+----------+ +| fec2fd0a-a4ef-4064-a292-95e9da68025c | e7d70ddf-9be0-4681-9456-aa8dce515e0e | full | 933 | 186 | 2015-03-12 09:46:40.818691 | finished | ++--------------------------------------+--------------------------------------+----------+-------+----------+----------------------------+----------+ + +Tests: + ++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+--------+ +| name | time | status | ++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+--------+ +| tearDownClass (tempest.api.image.v1.test_images.CreateRegisterImagesTest) | 0.0 | FAIL | +| tearDownClass (tempest.api.image.v1.test_images.UpdateImageMetaTest) | 0.0 | FAIL | +[...] +| tempest.cli.simple_read_only.volume.test_cinder.SimpleReadOnlyCinderClientTest.test_cinder_quota_show[id-18166673-ffa8-4df3-b60c-6375532288bc] | 1.309555 | OK | +| tempest.cli.simple_read_only.volume.test_cinder.SimpleReadOnlyCinderClientTest.test_cinder_rate_limits[id-b2c66ed9-ca96-4dc4-94cc-8083e664e516] | 1.277704 | OK | +| tempest.cli.simple_read_only.volume.test_cinder.SimpleReadOnlyCinderClientTest.test_cinder_region_list[id-95a2850c-35b4-4159-bb93-51647a5ad232] | 1.105877 | FAIL | +| tempest.cli.simple_read_only.volume.test_cinder.SimpleReadOnlyCinderClientTest.test_cinder_retries_list[id-6d97fcd2-5dd1-429d-af70-030c949d86cd] | 1.306407 | OK | +| tempest.cli.simple_read_only.volume.test_cinder.SimpleReadOnlyCinderClientTest.test_cinder_service_list[id-301b5ae1-9591-4e9f-999c-d525a9bdf822] | 1.24909 | OK | +| tempest.cli.simple_read_only.volume.test_cinder.SimpleReadOnlyCinderClientTest.test_cinder_snapshot_list[id-7a19955b-807c-481a-a2ee-9d76733eac28] | 1.270242 | OK | +[...] +| tempest.thirdparty.boto.test_s3_ec2_images.S3ImagesTest | 0.0 | SKIP | +| tempest.thirdparty.boto.test_s3_objects.S3BucketsTest.test_create_get_delete_object[id-4eea567a-b46a-405b-a475-6097e1faebde] | 0.239222 | FAIL | ++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+--------+ + +``` +Rally includes a reporting tool +https://rally.readthedocs.org/en/latest/tutorial/step_1_setting_up_env_and_running_benchmark_from_samples.html + + + + + +## Test description + +### Rally + +By default, the different Rally Scenarios are: +```bash + +ls samples/tasks/scenarios/ +authenticate cinder dummy heat mistral nova README.rst sahara vm +ceilometer designate glance keystone neutron quotas requests tempest-do-not-run-against-production zaqar + +``` + +tempest tests can be retrieved at https://github.com/openstack/tempest + + +## Automation -- cgit 1.2.3-korg