diff options
| author |   Juha Kosonen <juha.kosonen@nokia.com> | 2015-10-02 13:49:48 +0300 |
|---|---|---|
| committer | Juha Kosonen <juha.kosonen@nokia.com> | 2015-10-02 14:08:24 +0300 |
| commit | fe2e0ed59aa12d9f4bb30455e0c401c4be21f3ab (patch) | |
| tree | 77630a48c7790fc22a714ea65b39780d1cfb6daf /testcases | |
| parent | 96bf9abe9b1b26a79dcc86900e8eb33d8544e773 (diff) | |
Add SLA to Rally test cases
Set success criteria for Rally scenarios to indicate failure in case of a
failed iteration.
JIRA: FUNCTEST-58
Change-Id: Ie864ebfddf4be5f67c0af0ceca0d7d1d0c07cd19
Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com>
Diffstat (limited to 'testcases')
13 files changed, 255 insertions, 0 deletions
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json b/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json index 3ded19963..8bb589524 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json @@ -11,6 +11,9 @@ "tenants": 3, "users_per_tenant": 50 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -29,6 +32,9 @@ "tenants": 3, "users_per_tenant": 5 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -47,6 +53,9 @@ "tenants": 3, "users_per_tenant": 5 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -65,6 +74,9 @@ "tenants": 3, "users_per_tenant": 5 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -83,6 +95,9 @@ "tenants": 3, "users_per_tenant": 5 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -101,6 +116,9 @@ "tenants": 3, "users_per_tenant": 5 } + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json b/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json index 4fc3f769b..bada44733 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json @@ -20,6 +20,9 @@ "tenants": 2, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -41,6 +44,9 @@ "volumes": { "size": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -59,6 +65,9 @@ "tenants": 2, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -78,6 +87,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -100,6 +112,9 @@ "volumes": { "size": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -119,6 +134,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -141,6 +159,9 @@ "tenants": 2, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -162,6 +183,9 @@ "tenants": 2, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -183,6 +207,9 @@ "volumes": { "size": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -217,6 +244,9 @@ }, "servers_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -248,6 +278,9 @@ }, "servers_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -266,6 +299,9 @@ "tenants": 2, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -288,6 +324,9 @@ "size": 1, "volumes_per_tenant": 4 } + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-glance.json b/testcases/VIM/OpenStack/CI/suites/opnfv-glance.json index 68d561345..e905ccaba 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-glance.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-glance.json @@ -17,6 +17,9 @@ "image_container": "bare", "images_per_tenant": 4 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -37,6 +40,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -57,6 +63,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -81,6 +90,9 @@ "tenants": 3, "users_per_tenant": 5 } + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json b/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json index 2fbb70fba..a712afefe 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json @@ -11,6 +11,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -29,6 +32,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -47,6 +53,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -62,6 +71,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -80,6 +92,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -98,6 +113,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -117,6 +135,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -136,6 +157,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -155,6 +179,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -174,6 +201,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -193,6 +223,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -212,6 +245,9 @@ "tenants": 2, "users_per_tenant": 3 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -231,6 +267,9 @@ "stacks_per_tenant": 2, "resources_per_stack": 10 } + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json b/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json index 390a1ae13..f7291ed59 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json @@ -8,6 +8,9 @@ "type": "constant", "times": 100, "concurrency": 10 + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -20,6 +23,9 @@ "type": "constant", "times": 10, "concurrency": 1 + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -32,6 +38,9 @@ "type": "constant", "times": 100, "concurrency": 10 + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -44,6 +53,9 @@ "type": "constant", "times": 100, "concurrency": 10 + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -56,6 +68,9 @@ "type": "constant", "times": 100, "concurrency": 10 + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -69,6 +84,9 @@ "type": "constant", "times": 10, "concurrency": 10 + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json b/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json index 5d176ca0d..9fcdf583d 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json @@ -19,6 +19,9 @@ "network": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -45,6 +48,9 @@ "port": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -74,6 +80,9 @@ "router": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -101,6 +110,9 @@ "subnet": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -124,6 +136,9 @@ "network": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -150,6 +165,9 @@ "port": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -179,6 +197,9 @@ "router": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -206,6 +227,9 @@ "subnet": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -233,6 +257,9 @@ "network": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -265,6 +292,9 @@ "port": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -298,6 +328,9 @@ "router": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -329,6 +362,9 @@ "subnet": -1 } } + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json b/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json index 09ca4108a..e32fd57ec 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json @@ -20,6 +20,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -46,6 +49,9 @@ "network": { "start_cidr": "100.1.0.0/26" } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -74,6 +80,9 @@ "network": { "start_cidr": "100.1.0.0/26" } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -98,6 +107,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -122,6 +134,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -145,6 +160,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -175,6 +193,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -200,6 +221,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -224,6 +248,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -247,6 +274,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -272,6 +302,9 @@ "tenants": 2, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -298,6 +331,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -322,6 +358,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -337,6 +376,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -356,6 +398,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -371,6 +416,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -390,6 +438,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -417,6 +468,9 @@ }, "servers_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -445,6 +499,9 @@ "tenants": 1, "users_per_tenant": 1 } + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json b/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json index 1778a8dd0..1cc1855e7 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json @@ -14,6 +14,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -32,6 +35,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -50,6 +56,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -68,6 +77,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -86,6 +98,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-requests.json b/testcases/VIM/OpenStack/CI/suites/opnfv-requests.json index ead457dbf..4468d60a3 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-requests.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-requests.json @@ -11,6 +11,9 @@ "type": "constant", "times": 20, "concurrency": 5 + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -26,6 +29,9 @@ "type": "constant", "times": 20, "concurrency": 5 + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-smoke-green.json b/testcases/VIM/OpenStack/CI/suites/opnfv-smoke-green.json index a7eb345b7..b327b53e5 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-smoke-green.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-smoke-green.json @@ -230,6 +230,9 @@ "type": "serial", "times": 1, "concurrency": 1 + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-smoke.json b/testcases/VIM/OpenStack/CI/suites/opnfv-smoke.json index 8c40fab1d..31514d274 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-smoke.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-smoke.json @@ -265,6 +265,9 @@ "type": "serial", "times": 1, "concurrency": 1 + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json b/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json index 0b63070a0..b94de47e4 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json @@ -6,6 +6,9 @@ "type": "constant", "times": 1, "concurrency": 1 + }, + "sla": { + "failure_rate": {"max": 0} } } ] diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json b/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json index f50cf2b25..382f40251 100644 --- a/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json +++ b/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json @@ -26,6 +26,9 @@ }, "network": { } + }, + "sla": { + "failure_rate": {"max": 0} } } ], @@ -59,6 +62,9 @@ "tenants": 3, "users_per_tenant": 2 } + }, + "sla": { + "failure_rate": {"max": 0} } } ] |