Add SLA to Rally test cases

Set success criteria for Rally scenarios to indicate failure in case of a
failed iteration.

JIRA: FUNCTEST-58

Change-Id: Ie864ebfddf4be5f67c0af0ceca0d7d1d0c07cd19
Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com>

13 files changed, 255 insertions, 0 deletions

diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json b/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json
index 3ded19963..8bb589524 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-authenticate.json
@@ -11,6 +11,9 @@
                     "tenants": 3,
                     "users_per_tenant": 50
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -29,6 +32,9 @@
                     "tenants": 3,
                     "users_per_tenant": 5
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -47,6 +53,9 @@
                     "tenants": 3,
                     "users_per_tenant": 5
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -65,6 +74,9 @@
                     "tenants": 3,
                     "users_per_tenant": 5
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -83,6 +95,9 @@
                     "tenants": 3,
                     "users_per_tenant": 5
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -101,6 +116,9 @@
                     "tenants": 3,
                     "users_per_tenant": 5
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json b/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json
index 4fc3f769b..bada44733 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-cinder.json
@@ -20,6 +20,9 @@
                     "tenants": 2,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -41,6 +44,9 @@
                 "volumes": {
                     "size": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -59,6 +65,9 @@
                     "tenants": 2,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -78,6 +87,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -100,6 +112,9 @@
                 "volumes": {
                     "size": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -119,6 +134,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -141,6 +159,9 @@
                     "tenants": 2,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -162,6 +183,9 @@
                     "tenants": 2,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -183,6 +207,9 @@
                 "volumes": {
                     "size": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -217,6 +244,9 @@
                     },
                     "servers_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -248,6 +278,9 @@
                     },
                     "servers_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -266,6 +299,9 @@
                     "tenants": 2,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -288,6 +324,9 @@
                     "size": 1,
                     "volumes_per_tenant": 4
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-glance.json b/testcases/VIM/OpenStack/CI/suites/opnfv-glance.json
index 68d561345..e905ccaba 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-glance.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-glance.json
@@ -17,6 +17,9 @@
                     "image_container": "bare",
                     "images_per_tenant": 4
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -37,6 +40,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -57,6 +63,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -81,6 +90,9 @@
                     "tenants": 3,
                     "users_per_tenant": 5
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json b/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json
index 2fbb70fba..a712afefe 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-heat.json
@@ -11,6 +11,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -29,6 +32,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -47,6 +53,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -62,6 +71,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -80,6 +92,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -98,6 +113,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -117,6 +135,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -136,6 +157,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -155,6 +179,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -174,6 +201,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -193,6 +223,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -212,6 +245,9 @@
                     "tenants": 2,
                     "users_per_tenant": 3
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -231,6 +267,9 @@
                     "stacks_per_tenant": 2,
                     "resources_per_stack": 10
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json b/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json
index 390a1ae13..f7291ed59 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-keystone.json
@@ -8,6 +8,9 @@
                 "type": "constant",
                 "times": 100,
                 "concurrency": 10
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -20,6 +23,9 @@
                 "type": "constant",
                 "times": 10,
                 "concurrency": 1
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -32,6 +38,9 @@
                 "type": "constant",
                 "times": 100,
                 "concurrency": 10
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -44,6 +53,9 @@
                 "type": "constant",
                 "times": 100,
                 "concurrency": 10
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -56,6 +68,9 @@
                 "type": "constant",
                 "times": 100,
                 "concurrency": 10
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -69,6 +84,9 @@
                 "type": "constant",
                 "times": 10,
                 "concurrency": 10
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json b/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json
index 5d176ca0d..9fcdf583d 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-neutron.json
@@ -19,6 +19,9 @@
                         "network": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -45,6 +48,9 @@
                         "port": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -74,6 +80,9 @@
                         "router": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -101,6 +110,9 @@
                         "subnet": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -124,6 +136,9 @@
                         "network": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -150,6 +165,9 @@
                         "port": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -179,6 +197,9 @@
                         "router": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -206,6 +227,9 @@
                         "subnet": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -233,6 +257,9 @@
                         "network": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -265,6 +292,9 @@
                         "port": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -298,6 +328,9 @@
                         "router": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -329,6 +362,9 @@
                         "subnet": -1
                     }
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json b/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json
index 09ca4108a..e32fd57ec 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-nova.json
@@ -20,6 +20,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -46,6 +49,9 @@
                 "network": {
                     "start_cidr": "100.1.0.0/26"
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -74,6 +80,9 @@
                 "network": {
                     "start_cidr": "100.1.0.0/26"
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -98,6 +107,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -122,6 +134,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -145,6 +160,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -175,6 +193,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -200,6 +221,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -224,6 +248,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -247,6 +274,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -272,6 +302,9 @@
                     "tenants": 2,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -298,6 +331,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -322,6 +358,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -337,6 +376,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -356,6 +398,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -371,6 +416,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -390,6 +438,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -417,6 +468,9 @@
                     },
                     "servers_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -445,6 +499,9 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json b/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json
index 1778a8dd0..1cc1855e7 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-quotas.json
@@ -14,6 +14,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -32,6 +35,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -50,6 +56,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -68,6 +77,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -86,6 +98,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-requests.json b/testcases/VIM/OpenStack/CI/suites/opnfv-requests.json
index ead457dbf..4468d60a3 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-requests.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-requests.json
@@ -11,6 +11,9 @@
                 "type": "constant",
                 "times": 20,
                 "concurrency": 5
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -26,6 +29,9 @@
                 "type": "constant",
                 "times": 20,
                 "concurrency": 5
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-smoke-green.json b/testcases/VIM/OpenStack/CI/suites/opnfv-smoke-green.json
index a7eb345b7..b327b53e5 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-smoke-green.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-smoke-green.json
@@ -230,6 +230,9 @@
                 "type": "serial",
                 "times": 1,
                 "concurrency": 1
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-smoke.json b/testcases/VIM/OpenStack/CI/suites/opnfv-smoke.json
index 8c40fab1d..31514d274 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-smoke.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-smoke.json
@@ -265,6 +265,9 @@
                 "type": "serial",
                 "times": 1,
                 "concurrency": 1
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json b/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json
index 0b63070a0..b94de47e4 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-tempest.json
@@ -6,6 +6,9 @@
                 "type": "constant",
                 "times": 1,
                 "concurrency": 1
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]
diff --git a/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json b/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json
index f50cf2b25..382f40251 100644
--- a/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json
+++ b/testcases/VIM/OpenStack/CI/suites/opnfv-vm.json
@@ -26,6 +26,9 @@
                 },
                 "network": {
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ],
@@ -59,6 +62,9 @@
                     "tenants": 3,
                     "users_per_tenant": 2
                 }
+            },
+            "sla": {
+                "failure_rate": {"max": 0}
             }
         }
     ]