summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xci/run_tests.py25
-rw-r--r--ci/testcases.yaml18
-rw-r--r--ci/tier_builder.py5
-rw-r--r--ci/tier_handler.py9
4 files changed, 46 insertions, 11 deletions
diff --git a/ci/run_tests.py b/ci/run_tests.py
index 0fb850130..cb05d435d 100755
--- a/ci/run_tests.py
+++ b/ci/run_tests.py
@@ -45,6 +45,10 @@ EXEC_SCRIPT = ("%sci/exec_test.sh" % FUNCTEST_REPO)
CLEAN_FLAG = True
REPORT_FLAG = False
+# This will be the return code of this script. If any of the tests fails,
+# this variable will change to -1
+OVERALL_RESULT = 0
+
def print_separator(str, count=45):
line = ""
@@ -71,6 +75,7 @@ def cleanup():
def run_test(test):
+ global OVERALL_RESULT
start = datetime.datetime.now()
test_name = test.get_name()
logger.info("\n") # blank line
@@ -91,20 +96,24 @@ def run_test(test):
result = ft_utils.execute_command(cmd, logger, exit_on_error=False)
- if result != 0:
- logger.error("The test case '%s' failed. Cleaning and exiting."
- % test_name)
- if CLEAN_FLAG:
- cleanup()
- sys.exit(1)
-
if CLEAN_FLAG:
cleanup()
+
end = datetime.datetime.now()
duration = (end - start).seconds
str = ("%02d:%02d" % divmod(duration, 60))
logger.info("Test execution time: %s" % str)
+ if result != 0:
+ logger.error("The test case '%s' failed. " % test_name)
+ OVERALL_RESULT = -1
+
+ if test.get_blocking():
+ logger.info("This test case is blocking. Exiting...")
+ sys.exit(OVERALL_RESULT)
+
+ return result
+
def run_tier(tier):
tests = tier.get_tests()
@@ -181,7 +190,7 @@ def main():
else:
run_all(_tiers)
- sys.exit(0)
+ sys.exit(OVERALL_RESULT)
if __name__ == '__main__':
main()
diff --git a/ci/testcases.yaml b/ci/testcases.yaml
index e3c1e51da..11d4637fc 100644
--- a/ci/testcases.yaml
+++ b/ci/testcases.yaml
@@ -10,6 +10,7 @@ tiers:
-
name: healthcheck
criteria: 'status == "PASS"'
+ blocking: true
description: >-
This test case verifies the basic OpenStack services like
Keystone, Glance, Cinder, Neutron and Nova.
@@ -28,6 +29,7 @@ tiers:
-
name: vping_ssh
criteria: 'status == "PASS"'
+ blocking: true
description: >-
This test case verifies: 1) SSH to an instance using floating
IPs over the public network. 2) Connectivity between 2 instances
@@ -39,6 +41,7 @@ tiers:
-
name: vping_userdata
criteria: 'status == "PASS"'
+ blocking: true
description: >-
This test case verifies: 1) Boot a VM with given userdata.
2) Connectivity between 2 instances over a private network.
@@ -49,6 +52,7 @@ tiers:
-
name: tempest_smoke_serial
criteria: 'success_rate == 100%'
+ blocking: false
description: >-
This test case runs the smoke subset of the OpenStack
Tempest suite. The list of test cases is generated by
@@ -61,6 +65,7 @@ tiers:
-
name: rally_sanity
criteria: 'success_rate == 100%'
+ blocking: false
description: >-
This test case runs a sub group of tests of the OpenStack
Rally suite in smoke mode.
@@ -79,6 +84,7 @@ tiers:
-
name: odl
criteria: 'success_rate == 100%'
+ blocking: true
description: >-
Test Suite for the OpenDaylight SDN Controller. It integrates
some test suites from upstream using Robot as the test
@@ -90,6 +96,7 @@ tiers:
-
name: onos
criteria: 'status == "PASS"'
+ blocking: true
description: >-
Test Suite for the ONOS SDN Controller. It integrates
some test suites from upstream using TestON as the test
@@ -109,6 +116,7 @@ tiers:
-
name: promise
criteria: 'success_rate == 100%'
+ blocking: false
description: >-
Test suite from Promise project.
dependencies:
@@ -118,6 +126,7 @@ tiers:
-
name: doctor
criteria: 'status == "PASS"'
+ blocking: false
description: >-
Test suite from Doctor project.
dependencies:
@@ -127,6 +136,7 @@ tiers:
-
name: bgpvpn
criteria: 'status == "PASS"'
+ blocking: false
description: >-
Test suite from SDNVPN project.
dependencies:
@@ -136,6 +146,7 @@ tiers:
-
name: security_scan
criteria: 'status == "PASS"'
+ blocking: false
description: >-
Simple security Scan
dependencies:
@@ -145,6 +156,7 @@ tiers:
-
name: copper
criteria: 'status == "PASS"'
+ blocking: false
description: >-
Test suite for policy management based on OpenStack Congress
dependencies:
@@ -153,6 +165,7 @@ tiers:
-
name: moon
criteria: 'status == "PASS"'
+ blocking: false
description: >-
Security management system for OPNFV
dependencies:
@@ -161,6 +174,7 @@ tiers:
-
name: multisite
criteria: 'status == "PASS"'
+ blocking: false
description: >-
Test suite from kingbird
dependencies:
@@ -169,6 +183,7 @@ tiers:
-
name: domino
criteria: 'status == "PASS"'
+ blocking: false
description: >-
Test suite for template distribution based on Domino
dependencies:
@@ -184,6 +199,7 @@ tiers:
-
name: tempest_full_parallel
criteria: 'success_rate >= 80%'
+ blocking: false
description: >-
The list of test cases is generated by
Tempest automatically and depends on the parameters of
@@ -195,6 +211,7 @@ tiers:
-
name: rally_full
criteria: 'success_rate >= 90%'
+ blocking: false
description: >-
This test case runs the full suite of scenarios of the OpenStack
Rally suite using several threads and iterations.
@@ -212,6 +229,7 @@ tiers:
-
name: vims
criteria: 'status == "PASS"'
+ blocking: false
description: >-
This test case deploys an OpenSource vIMS solution from Clearwater
using the Cloudify orchestrator. It also runs some signaling traffic.
diff --git a/ci/tier_builder.py b/ci/tier_builder.py
index b9357bc2d..e1c3e49e6 100644
--- a/ci/tier_builder.py
+++ b/ci/tier_builder.py
@@ -7,12 +7,12 @@
# http://www.apache.org/licenses/LICENSE-2.0
#
-import yaml
-
import tier_handler as th
+import yaml
class TierBuilder:
+
def __init__(self, ci_installer, ci_scenario, testcases_file):
self.ci_installer = ci_installer
self.ci_scenario = ci_scenario
@@ -49,6 +49,7 @@ class TierBuilder:
testcase = th.TestCase(name=dic_testcase['name'],
dependency=dep,
criteria=dic_testcase['criteria'],
+ blocking=dic_testcase['blocking'],
description=dic_testcase['description'])
if testcase.is_compatible(self.ci_installer, self.ci_scenario):
tier.add_test(testcase)
diff --git a/ci/tier_handler.py b/ci/tier_handler.py
index 20a47e997..27b9cbcfd 100644
--- a/ci/tier_handler.py
+++ b/ci/tier_handler.py
@@ -29,6 +29,7 @@ def split_text(text, max_len):
class Tier:
+
def __init__(self, name, order, ci_loop, description=""):
self.tests_array = []
self.name = name
@@ -102,11 +103,13 @@ class Tier:
class TestCase:
- def __init__(self, name, dependency, criteria, description=""):
+
+ def __init__(self, name, dependency, criteria, blocking, description=""):
self.name = name
self.dependency = dependency
self.description = description
self.criteria = criteria
+ self.blocking = blocking
def is_compatible(self, ci_installer, ci_scenario):
try:
@@ -128,6 +131,9 @@ class TestCase:
def get_criteria(self):
return self.criteria
+ def get_blocking(self):
+ return self.blocking
+
def __str__(self):
lines = split_text(self.description, LINE_LENGTH - 6)
@@ -151,6 +157,7 @@ class TestCase:
class Dependency:
+
def __init__(self, installer, scenario):
self.installer = installer
self.scenario = scenario