diff options
| -rwxr-xr-x | ci/run_tests.py | 25 | ||||
| -rw-r--r-- | ci/testcases.yaml | 18 | ||||
| -rw-r--r-- | ci/tier_builder.py | 5 | ||||
| -rw-r--r-- | ci/tier_handler.py | 9 |
4 files changed, 46 insertions, 11 deletions
diff --git a/ci/run_tests.py b/ci/run_tests.py index 0fb85013..cb05d435 100755 --- a/ci/run_tests.py +++ b/ci/run_tests.py @@ -45,6 +45,10 @@ EXEC_SCRIPT = ("%sci/exec_test.sh" % FUNCTEST_REPO) CLEAN_FLAG = True REPORT_FLAG = False +# This will be the return code of this script. If any of the tests fails, +# this variable will change to -1 +OVERALL_RESULT = 0 + def print_separator(str, count=45): line = "" @@ -71,6 +75,7 @@ def cleanup(): def run_test(test): + global OVERALL_RESULT start = datetime.datetime.now() test_name = test.get_name() logger.info("\n") # blank line @@ -91,20 +96,24 @@ def run_test(test): result = ft_utils.execute_command(cmd, logger, exit_on_error=False) - if result != 0: - logger.error("The test case '%s' failed. Cleaning and exiting." - % test_name) - if CLEAN_FLAG: - cleanup() - sys.exit(1) - if CLEAN_FLAG: cleanup() + end = datetime.datetime.now() duration = (end - start).seconds str = ("%02d:%02d" % divmod(duration, 60)) logger.info("Test execution time: %s" % str) + if result != 0: + logger.error("The test case '%s' failed. " % test_name) + OVERALL_RESULT = -1 + + if test.get_blocking(): + logger.info("This test case is blocking. Exiting...") + sys.exit(OVERALL_RESULT) + + return result + def run_tier(tier): tests = tier.get_tests() @@ -181,7 +190,7 @@ def main(): else: run_all(_tiers) - sys.exit(0) + sys.exit(OVERALL_RESULT) if __name__ == '__main__': main() diff --git a/ci/testcases.yaml b/ci/testcases.yaml index e3c1e51d..11d4637f 100644 --- a/ci/testcases.yaml +++ b/ci/testcases.yaml @@ -10,6 +10,7 @@ tiers: - name: healthcheck criteria: 'status == "PASS"' + blocking: true description: >- This test case verifies the basic OpenStack services like Keystone, Glance, Cinder, Neutron and Nova. @@ -28,6 +29,7 @@ tiers: - name: vping_ssh criteria: 'status == "PASS"' + blocking: true description: >- This test case verifies: 1) SSH to an instance using floating IPs over the public network. 2) Connectivity between 2 instances @@ -39,6 +41,7 @@ tiers: - name: vping_userdata criteria: 'status == "PASS"' + blocking: true description: >- This test case verifies: 1) Boot a VM with given userdata. 2) Connectivity between 2 instances over a private network. @@ -49,6 +52,7 @@ tiers: - name: tempest_smoke_serial criteria: 'success_rate == 100%' + blocking: false description: >- This test case runs the smoke subset of the OpenStack Tempest suite. The list of test cases is generated by @@ -61,6 +65,7 @@ tiers: - name: rally_sanity criteria: 'success_rate == 100%' + blocking: false description: >- This test case runs a sub group of tests of the OpenStack Rally suite in smoke mode. @@ -79,6 +84,7 @@ tiers: - name: odl criteria: 'success_rate == 100%' + blocking: true description: >- Test Suite for the OpenDaylight SDN Controller. It integrates some test suites from upstream using Robot as the test @@ -90,6 +96,7 @@ tiers: - name: onos criteria: 'status == "PASS"' + blocking: true description: >- Test Suite for the ONOS SDN Controller. It integrates some test suites from upstream using TestON as the test @@ -109,6 +116,7 @@ tiers: - name: promise criteria: 'success_rate == 100%' + blocking: false description: >- Test suite from Promise project. dependencies: @@ -118,6 +126,7 @@ tiers: - name: doctor criteria: 'status == "PASS"' + blocking: false description: >- Test suite from Doctor project. dependencies: @@ -127,6 +136,7 @@ tiers: - name: bgpvpn criteria: 'status == "PASS"' + blocking: false description: >- Test suite from SDNVPN project. dependencies: @@ -136,6 +146,7 @@ tiers: - name: security_scan criteria: 'status == "PASS"' + blocking: false description: >- Simple security Scan dependencies: @@ -145,6 +156,7 @@ tiers: - name: copper criteria: 'status == "PASS"' + blocking: false description: >- Test suite for policy management based on OpenStack Congress dependencies: @@ -153,6 +165,7 @@ tiers: - name: moon criteria: 'status == "PASS"' + blocking: false description: >- Security management system for OPNFV dependencies: @@ -161,6 +174,7 @@ tiers: - name: multisite criteria: 'status == "PASS"' + blocking: false description: >- Test suite from kingbird dependencies: @@ -169,6 +183,7 @@ tiers: - name: domino criteria: 'status == "PASS"' + blocking: false description: >- Test suite for template distribution based on Domino dependencies: @@ -184,6 +199,7 @@ tiers: - name: tempest_full_parallel criteria: 'success_rate >= 80%' + blocking: false description: >- The list of test cases is generated by Tempest automatically and depends on the parameters of @@ -195,6 +211,7 @@ tiers: - name: rally_full criteria: 'success_rate >= 90%' + blocking: false description: >- This test case runs the full suite of scenarios of the OpenStack Rally suite using several threads and iterations. @@ -212,6 +229,7 @@ tiers: - name: vims criteria: 'status == "PASS"' + blocking: false description: >- This test case deploys an OpenSource vIMS solution from Clearwater using the Cloudify orchestrator. It also runs some signaling traffic. diff --git a/ci/tier_builder.py b/ci/tier_builder.py index b9357bc2..e1c3e49e 100644 --- a/ci/tier_builder.py +++ b/ci/tier_builder.py @@ -7,12 +7,12 @@ # http://www.apache.org/licenses/LICENSE-2.0 # -import yaml - import tier_handler as th +import yaml class TierBuilder: + def __init__(self, ci_installer, ci_scenario, testcases_file): self.ci_installer = ci_installer self.ci_scenario = ci_scenario @@ -49,6 +49,7 @@ class TierBuilder: testcase = th.TestCase(name=dic_testcase['name'], dependency=dep, criteria=dic_testcase['criteria'], + blocking=dic_testcase['blocking'], description=dic_testcase['description']) if testcase.is_compatible(self.ci_installer, self.ci_scenario): tier.add_test(testcase) diff --git a/ci/tier_handler.py b/ci/tier_handler.py index 20a47e99..27b9cbcf 100644 --- a/ci/tier_handler.py +++ b/ci/tier_handler.py @@ -29,6 +29,7 @@ def split_text(text, max_len): class Tier: + def __init__(self, name, order, ci_loop, description=""): self.tests_array = [] self.name = name @@ -102,11 +103,13 @@ class Tier: class TestCase: - def __init__(self, name, dependency, criteria, description=""): + + def __init__(self, name, dependency, criteria, blocking, description=""): self.name = name self.dependency = dependency self.description = description self.criteria = criteria + self.blocking = blocking def is_compatible(self, ci_installer, ci_scenario): try: @@ -128,6 +131,9 @@ class TestCase: def get_criteria(self): return self.criteria + def get_blocking(self): + return self.blocking + def __str__(self): lines = split_text(self.description, LINE_LENGTH - 6) @@ -151,6 +157,7 @@ class TestCase: class Dependency: + def __init__(self, installer, scenario): self.installer = installer self.scenario = scenario |