aboutsummaryrefslogtreecommitdiffstats
path: root/functest/ci
diff options
context:
space:
mode:
authorCédric Ollivier <cedric.ollivier@orange.com>2017-04-24 08:58:20 +0200
committerCédric Ollivier <cedric.ollivier@orange.com>2017-04-26 09:35:47 +0200
commit37c98a6fa101be2b32129dd5d71ad750f5245b88 (patch)
tree9b3f8700cb5af4054593c667a7f82aa160e32a8f /functest/ci
parente620488a6747318c40eb973c2607ae6d44e95b8f (diff)
Manage criteria in TestCase
It converts all criteria values to the corresponding percent in functest/ci/testcases.yaml. Result is expected to be equal or greater than criteria. If both are 0, result is considered as false. It is compatible with the old behavior but warns to update. It will allow a safer remove. It also fixes a bug in test_tempest to allow merging [1] and tier_handler.py which required that type criteria was str. [1] https://gerrit.opnfv.org/gerrit/#/c/27949/ Change-Id: Ib6edcfa3103b7d51b0bdc83119f1cea2a8be9fbc Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
Diffstat (limited to 'functest/ci')
-rw-r--r--functest/ci/testcases.yaml66
-rw-r--r--functest/ci/tier_handler.py2
2 files changed, 34 insertions, 34 deletions
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 39988306..8f2cc4bc 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -10,7 +10,7 @@ tiers:
-
case_name: connection_check
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: true
clean_flag: false
description: >-
@@ -30,7 +30,7 @@ tiers:
-
case_name: api_check
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: true
clean_flag: false
description: >-
@@ -49,7 +49,7 @@ tiers:
-
case_name: snaps_health_check
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: true
clean_flag: false
description: >-
@@ -73,7 +73,7 @@ tiers:
-
case_name: vping_ssh
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: true
clean_flag: true
description: >-
@@ -90,7 +90,7 @@ tiers:
-
case_name: vping_userdata
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: true
clean_flag: true
description: >-
@@ -106,7 +106,7 @@ tiers:
-
case_name: tempest_smoke_serial
project_name: functest
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -124,7 +124,7 @@ tiers:
-
case_name: rally_sanity
project_name: functest
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: false
clean_flag: false
description: >-
@@ -140,7 +140,7 @@ tiers:
-
case_name: refstack_defcore
project_name: functest
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -156,7 +156,7 @@ tiers:
-
case_name: odl
project_name: functest
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: true
clean_flag: false
description: >-
@@ -177,7 +177,7 @@ tiers:
-
case_name: odl_netvirt
project_name: functest
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: false
clean_flag: false
description: >-
@@ -200,7 +200,7 @@ tiers:
-
case_name: fds
project_name: functest
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: false
clean_flag: false
description: >-
@@ -220,7 +220,7 @@ tiers:
-
case_name: onos
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: true
clean_flag: true
description: >-
@@ -237,7 +237,7 @@ tiers:
-
case_name: snaps_smoke
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: false
description: >-
@@ -267,7 +267,7 @@ tiers:
-
case_name: promise
project_name: promise
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -284,7 +284,7 @@ tiers:
-
case_name: doctor-notification
project_name: doctor
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -301,7 +301,7 @@ tiers:
-
case_name: bgpvpn
project_name: sdnvpn
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -318,7 +318,7 @@ tiers:
-
case_name: security_scan
project_name: securityscanning
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -335,7 +335,7 @@ tiers:
-
case_name: copper
project_name: copper
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -352,7 +352,7 @@ tiers:
-
case_name: multisite
project_name: multisite
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: false
clean_flag: false
description: >-
@@ -366,7 +366,7 @@ tiers:
-
case_name: functest-odl-sfc
project_name: sfc
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -382,7 +382,7 @@ tiers:
-
case_name: onos_sfc
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: true
clean_flag: true
description: >-
@@ -396,7 +396,7 @@ tiers:
-
case_name: parser-basics
project_name: parser
- criteria: 'ret == 0'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -412,7 +412,7 @@ tiers:
-
case_name: domino-multinode
project_name: domino
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -428,7 +428,7 @@ tiers:
-
case_name: gluon_vping
project_name: netready
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -444,7 +444,7 @@ tiers:
-
case_name: barometercollectd
project_name: barometer
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -468,7 +468,7 @@ tiers:
-
case_name: tempest_full_parallel
project_name: functest
- criteria: 'success_rate >= 80%'
+ criteria: 80
blocking: false
clean_flag: true
description: >-
@@ -484,7 +484,7 @@ tiers:
-
case_name: tempest_custom
project_name: functest
- criteria: 'success_rate == 100%'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -502,7 +502,7 @@ tiers:
-
case_name: rally_full
project_name: functest
- criteria: 'success_rate >= 90%'
+ criteria: 90
blocking: false
clean_flag: false
description: >-
@@ -525,7 +525,7 @@ tiers:
-
case_name: cloudify_ims
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -540,7 +540,7 @@ tiers:
# -
# case_name: aaa
# project_name: functest
-# criteria: 'ret == 0'
+# criteria: 100
# blocking: false
# clean_flag: true
# description: >-
@@ -554,7 +554,7 @@ tiers:
-
case_name: orchestra_ims
project_name: functest
- criteria: 'ret == 0'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -569,7 +569,7 @@ tiers:
-
case_name: opera-vims
project_name: opera
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
@@ -584,7 +584,7 @@ tiers:
-
case_name: vyos_vrouter
project_name: functest
- criteria: 'status == "PASS"'
+ criteria: 100
blocking: false
clean_flag: true
description: >-
diff --git a/functest/ci/tier_handler.py b/functest/ci/tier_handler.py
index 6b4864b5..fe7372a3 100644
--- a/functest/ci/tier_handler.py
+++ b/functest/ci/tier_handler.py
@@ -158,7 +158,7 @@ class TestCase(object):
for line in lines:
out += ("| " + line.ljust(LINE_LENGTH - 7) + " |\n")
out += ("| Criteria: " +
- self.criteria.ljust(LINE_LENGTH - 14) + "|\n")
+ str(self.criteria).ljust(LINE_LENGTH - 14) + "|\n")
out += ("| Dependencies:".ljust(LINE_LENGTH - 1) + "|\n")
installer = self.dependency.get_installer()
scenario = self.dependency.get_scenario()