Merge "Create a flag in the tests to indicate if they should be blocking or not"

4 files changed, 46 insertions, 11 deletions

diff --git a/ci/run_tests.py b/ci/run_tests.py
index 0fb850130..cb05d435d 100755
--- a/ci/run_tests.py
+++ b/ci/run_tests.py
@@ -45,6 +45,10 @@ EXEC_SCRIPT = ("%sci/exec_test.sh" % FUNCTEST_REPO)
 CLEAN_FLAG = True
 REPORT_FLAG = False
 
+# This will be the return code of this script. If any of the tests fails,
+# this variable will change to -1
+OVERALL_RESULT = 0
+
 
 def print_separator(str, count=45):
     line = ""
@@ -71,6 +75,7 @@ def cleanup():
 
 
 def run_test(test):
+    global OVERALL_RESULT
     start = datetime.datetime.now()
     test_name = test.get_name()
     logger.info("\n")  # blank line
@@ -91,20 +96,24 @@ def run_test(test):
 
     result = ft_utils.execute_command(cmd, logger, exit_on_error=False)
 
-    if result != 0:
-        logger.error("The test case '%s' failed. Cleaning and exiting."
-                     % test_name)
-        if CLEAN_FLAG:
-            cleanup()
-        sys.exit(1)
-
     if CLEAN_FLAG:
         cleanup()
+
     end = datetime.datetime.now()
     duration = (end - start).seconds
     str = ("%02d:%02d" % divmod(duration, 60))
     logger.info("Test execution time: %s" % str)
 
+    if result != 0:
+        logger.error("The test case '%s' failed. " % test_name)
+        OVERALL_RESULT = -1
+
+        if test.get_blocking():
+            logger.info("This test case is blocking. Exiting...")
+            sys.exit(OVERALL_RESULT)
+
+    return result
+
 
 def run_tier(tier):
     tests = tier.get_tests()
@@ -181,7 +190,7 @@ def main():
     else:
         run_all(_tiers)
 
-    sys.exit(0)
+    sys.exit(OVERALL_RESULT)
 
 if __name__ == '__main__':
     main()
diff --git a/ci/testcases.yaml b/ci/testcases.yaml
index e3c1e51da..11d4637fc 100644
--- a/ci/testcases.yaml
+++ b/ci/testcases.yaml
@@ -10,6 +10,7 @@ tiers:
             -
                 name: healthcheck
                 criteria: 'status == "PASS"'
+                blocking: true
                 description: >-
                     This test case verifies the basic OpenStack services like
                     Keystone, Glance, Cinder, Neutron and Nova.
@@ -28,6 +29,7 @@ tiers:
             -
                 name: vping_ssh
                 criteria: 'status == "PASS"'
+                blocking: true
                 description: >-
                     This test case verifies: 1) SSH to an instance using floating
                     IPs over the public network. 2) Connectivity between 2 instances
@@ -39,6 +41,7 @@ tiers:
             -
                 name: vping_userdata
                 criteria: 'status == "PASS"'
+                blocking: true
                 description: >-
                     This test case verifies:  1) Boot a VM with given userdata.
                     2) Connectivity between 2 instances over a private network.
@@ -49,6 +52,7 @@ tiers:
             -
                 name: tempest_smoke_serial
                 criteria: 'success_rate == 100%'
+                blocking: false
                 description: >-
                     This test case runs the smoke subset of the OpenStack
                     Tempest suite. The list of test cases is generated by
@@ -61,6 +65,7 @@ tiers:
             -
                 name: rally_sanity
                 criteria: 'success_rate == 100%'
+                blocking: false
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Rally suite in smoke mode.
@@ -79,6 +84,7 @@ tiers:
             -
                 name: odl
                 criteria: 'success_rate == 100%'
+                blocking: true
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller. It integrates
                     some test suites from upstream using Robot as the test
@@ -90,6 +96,7 @@ tiers:
             -
                 name: onos
                 criteria: 'status == "PASS"'
+                blocking: true
                 description: >-
                     Test Suite for the ONOS SDN Controller. It integrates
                     some test suites from upstream using TestON as the test
@@ -109,6 +116,7 @@ tiers:
             -
                 name: promise
                 criteria: 'success_rate == 100%'
+                blocking: false
                 description: >-
                     Test suite from Promise project.
                 dependencies:
@@ -118,6 +126,7 @@ tiers:
             -
                 name: doctor
                 criteria: 'status == "PASS"'
+                blocking: false
                 description: >-
                     Test suite from Doctor project.
                 dependencies:
@@ -127,6 +136,7 @@ tiers:
             -
                 name: bgpvpn
                 criteria: 'status == "PASS"'
+                blocking: false
                 description: >-
                     Test suite from SDNVPN project.
                 dependencies:
@@ -136,6 +146,7 @@ tiers:
             -
                 name: security_scan
                 criteria: 'status == "PASS"'
+                blocking: false
                 description: >-
                     Simple security Scan
                 dependencies:
@@ -145,6 +156,7 @@ tiers:
             -
                 name: copper
                 criteria: 'status == "PASS"'
+                blocking: false
                 description: >-
                     Test suite for policy management based on OpenStack Congress
                 dependencies:
@@ -153,6 +165,7 @@ tiers:
             -
                 name: moon
                 criteria: 'status == "PASS"'
+                blocking: false
                 description: >-
                     Security management system for OPNFV
                 dependencies:
@@ -161,6 +174,7 @@ tiers:
             -
                 name: multisite
                 criteria: 'status == "PASS"'
+                blocking: false
                 description: >-
                     Test suite from kingbird
                 dependencies:
@@ -169,6 +183,7 @@ tiers:
             -
                 name: domino
                 criteria: 'status == "PASS"'
+                blocking: false
                 description: >-
                     Test suite for template distribution based on Domino
                 dependencies:
@@ -184,6 +199,7 @@ tiers:
             -
                 name: tempest_full_parallel
                 criteria: 'success_rate >= 80%'
+                blocking: false
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
@@ -195,6 +211,7 @@ tiers:
             -
                 name: rally_full
                 criteria: 'success_rate >= 90%'
+                blocking: false
                 description: >-
                     This test case runs the full suite of scenarios of the OpenStack
                     Rally suite using several threads and iterations.
@@ -212,6 +229,7 @@ tiers:
             -
                 name: vims
                 criteria: 'status == "PASS"'
+                blocking: false
                 description: >-
                     This test case deploys an OpenSource vIMS solution from Clearwater
                     using the Cloudify orchestrator. It also runs some signaling traffic.
diff --git a/ci/tier_builder.py b/ci/tier_builder.py
index b9357bc2d..e1c3e49e6 100644
--- a/ci/tier_builder.py
+++ b/ci/tier_builder.py
@@ -7,12 +7,12 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
-import yaml
-
 import tier_handler as th
+import yaml
 
 
 class TierBuilder:
+
     def __init__(self, ci_installer, ci_scenario, testcases_file):
         self.ci_installer = ci_installer
         self.ci_scenario = ci_scenario
@@ -49,6 +49,7 @@ class TierBuilder:
                 testcase = th.TestCase(name=dic_testcase['name'],
                                        dependency=dep,
                                        criteria=dic_testcase['criteria'],
+                                       blocking=dic_testcase['blocking'],
                                        description=dic_testcase['description'])
                 if testcase.is_compatible(self.ci_installer, self.ci_scenario):
                     tier.add_test(testcase)
diff --git a/ci/tier_handler.py b/ci/tier_handler.py
index 20a47e997..27b9cbcfd 100644
--- a/ci/tier_handler.py
+++ b/ci/tier_handler.py
@@ -29,6 +29,7 @@ def split_text(text, max_len):
 
 
 class Tier:
+
     def __init__(self, name, order, ci_loop, description=""):
         self.tests_array = []
         self.name = name
@@ -102,11 +103,13 @@ class Tier:
 
 
 class TestCase:
-    def __init__(self, name, dependency, criteria, description=""):
+
+    def __init__(self, name, dependency, criteria, blocking, description=""):
         self.name = name
         self.dependency = dependency
         self.description = description
         self.criteria = criteria
+        self.blocking = blocking
 
     def is_compatible(self, ci_installer, ci_scenario):
         try:
@@ -128,6 +131,9 @@ class TestCase:
     def get_criteria(self):
         return self.criteria
 
+    def get_blocking(self):
+        return self.blocking
+
     def __str__(self):
         lines = split_text(self.description, LINE_LENGTH - 6)
 
@@ -151,6 +157,7 @@ class TestCase:
 
 
 class Dependency:
+
     def __init__(self, installer, scenario):
         self.installer = installer
         self.scenario = scenario