Merge "Define create_snapshot() and clean() in TestCase"

4 files changed, 23 insertions, 66 deletions

diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py
index 9b64a0ce..af45c321 100755
--- a/functest/ci/run_tests.py
+++ b/functest/ci/run_tests.py
@@ -22,8 +22,6 @@ import prettytable
 import functest.ci.tier_builder as tb
 import functest.core.testcase as testcase
 import functest.utils.functest_utils as ft_utils
-import functest.utils.openstack_clean as os_clean
-import functest.utils.openstack_snapshot as os_snapshot
 import functest.utils.openstack_utils as os_utils
 from functest.utils.constants import CONST
 
@@ -98,14 +96,6 @@ class Runner(object):
                     CONST.__setattr__('OS_PASSWORD', value)
 
     @staticmethod
-    def generate_os_snapshot():
-        os_snapshot.main()
-
-    @staticmethod
-    def cleanup():
-        os_clean.main()
-
-    @staticmethod
     def get_run_dict(testname):
         try:
             dict = ft_utils.get_dict_by_test(testname)
@@ -124,14 +114,11 @@ class Runner(object):
                 "The test case {} is not enabled".format(test.get_name()))
         logger.info("\n")  # blank line
         self.print_separator("=")
-        logger.info("Running test case '%s'..." % test.get_name())
+        logger.info("Running test case '%s'...", test.get_name())
         self.print_separator("=")
         logger.debug("\n%s" % test)
         self.source_rc_file()
 
-        if test.needs_clean() and self.clean_flag:
-            self.generate_os_snapshot()
-
         flags = " -t %s" % test.get_name()
         if self.report_flag:
             flags += " -r"
@@ -145,6 +132,9 @@ class Runner(object):
                 test_dict = ft_utils.get_dict_by_test(test.get_name())
                 test_case = cls(**test_dict)
                 self.executed_test_cases.append(test_case)
+                if self.clean_flag:
+                    if test_case.create_snapshot() != test_case.EX_OK:
+                        return result
                 try:
                     kwargs = run_dict['args']
                     result = test_case.run(**kwargs)
@@ -155,6 +145,8 @@ class Runner(object):
                         test_case.push_to_db()
                     result = test_case.is_successful()
                 logger.info("Test result:\n\n%s\n", test_case)
+                if self.clean_flag:
+                    test_case.clean()
             except ImportError:
                 logger.exception("Cannot import module {}".format(
                     run_dict['module']))
@@ -164,15 +156,7 @@ class Runner(object):
         else:
             raise Exception("Cannot import the class for the test case.")
 
-        if test.needs_clean() and self.clean_flag:
-            self.cleanup()
-        if result != testcase.TestCase.EX_OK:
-            logger.error("The test case '%s' failed. " % test.get_name())
-            self.overall_result = Result.EX_ERROR
-            if test.is_blocking():
-                raise BlockingTestFailed(
-                    "The test case {} failed and is blocking".format(
-                        test.get_name()))
+        return result
 
     def run_tier(self, tier):
         tier_name = tier.get_name()
@@ -187,7 +171,14 @@ class Runner(object):
         self.print_separator("#")
         logger.debug("\n%s" % tier)
         for test in tests:
-            self.run_test(test, tier_name)
+            result = self.run_test(test, tier_name)
+            if result != testcase.TestCase.EX_OK:
+                logger.error("The test case '%s' failed.", test.get_name())
+                self.overall_result = Result.EX_ERROR
+                if test.is_blocking():
+                    raise BlockingTestFailed(
+                        "The test case {} failed and is blocking".format(
+                            test.get_name()))
 
     def run_all(self, tiers):
         summary = ""
@@ -225,9 +216,14 @@ class Runner(object):
                 if _tiers.get_tier(kwargs['test']):
                     self.run_tier(_tiers.get_tier(kwargs['test']))
                 elif _tiers.get_test(kwargs['test']):
-                    self.run_test(_tiers.get_test(kwargs['test']),
-                                  _tiers.get_tier_name(kwargs['test']),
-                                  kwargs['test'])
+                    result = self.run_test(
+                        _tiers.get_test(kwargs['test']),
+                        _tiers.get_tier_name(kwargs['test']),
+                        kwargs['test'])
+                    if result != testcase.TestCase.EX_OK:
+                        logger.error("The test case '%s' failed.",
+                                     kwargs['test'])
+                        self.overall_result = Result.EX_ERROR
                 elif kwargs['test'] == "all":
                     self.run_all(_tiers)
                 else:
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 8222df10..7d518324 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -12,7 +12,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: true
-                clean_flag: false
                 description: >-
                     This test case verifies the retrieval of OpenStack clients:
                     Keystone, Glance, Neutron and Nova and may perform some
@@ -31,7 +30,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: true
-                clean_flag: false
                 description: >-
                     This test case verifies the retrieval of OpenStack clients:
                     Keystone, Glance, Neutron and Nova and may perform some
@@ -50,7 +48,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: true
-                clean_flag: false
                 description: >-
                     This test case creates executes the SimpleHealthCheck
                     Python test class which creates an, image, flavor, network,
@@ -75,7 +72,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: true
-                clean_flag: true
                 description: >-
                     This test case verifies: 1) SSH to an instance using floating
                     IPs over the public network. 2) Connectivity between 2 instances
@@ -92,7 +88,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: true
-                clean_flag: true
                 description: >-
                     This test case verifies:  1) Boot a VM with given userdata.
                     2) Connectivity between 2 instances over a private network.
@@ -108,7 +103,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     This test case runs the smoke subset of the OpenStack
                     Tempest suite. The list of test cases is generated by
@@ -126,7 +120,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: false
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Rally suite in smoke mode.
@@ -142,7 +135,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Defcore testcases by using refstack client.
@@ -158,7 +150,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: true
-                clean_flag: false
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller. It
                     integrates some test suites from upstream using
@@ -179,7 +170,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: false
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller when
                     the NetVirt features are installed. It integrates
@@ -202,7 +192,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: false
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller when GBP features are
                     installed. It integrates some test suites from upstream using
@@ -222,7 +211,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: true
-                clean_flag: true
                 description: >-
                     Test Suite for the ONOS SDN Controller. It integrates
                     some test suites from upstream using TestON as the test
@@ -239,7 +227,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: false
                 description: >-
                     This test case contains tests that setup and destroy
                     environments with VMs with and without Floating IPs
@@ -269,7 +256,6 @@ tiers:
                 project_name: promise
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite from Promise project.
                 dependencies:
@@ -287,7 +273,6 @@ tiers:
                 project_name: doctor
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite from Doctor project.
                 dependencies:
@@ -304,7 +289,6 @@ tiers:
                 project_name: sdnvpn
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite from SDNVPN project.
                 dependencies:
@@ -322,7 +306,6 @@ tiers:
                 project_name: securityscanning
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Simple Security Scan
                 dependencies:
@@ -340,7 +323,6 @@ tiers:
                 project_name: copper
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite for policy management based on OpenStack Congress
                 dependencies:
@@ -358,7 +340,6 @@ tiers:
                 project_name: multisite
                 criteria: 100
                 blocking: false
-                clean_flag: false
                 description: >-
                     Test suite from kingbird
                 dependencies:
@@ -374,7 +355,6 @@ tiers:
                 project_name: sfc
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite for odl-sfc to test two chains and two SFs
                 dependencies:
@@ -392,7 +372,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: true
-                clean_flag: true
                 description: >-
                     Test Suite for onos-sfc to test sfc function.
                 dependencies:
@@ -408,7 +387,6 @@ tiers:
                 project_name: parser
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite from Parser project.
                 dependencies:
@@ -426,7 +404,6 @@ tiers:
                 project_name: domino
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite from Domino project.
                 dependencies:
@@ -444,7 +421,6 @@ tiers:
                 project_name: netready
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite from Netready project.
                 dependencies:
@@ -462,7 +438,6 @@ tiers:
                 project_name: barometer
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     Test suite for the Barometer project. Separate tests verify the
                     proper configuration and functionality of the following
@@ -487,7 +462,6 @@ tiers:
                 project_name: functest
                 criteria: 80
                 blocking: false
-                clean_flag: true
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
@@ -504,7 +478,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     The test case allows running a customized list of tempest
                     test cases defined in a file under
@@ -523,7 +496,6 @@ tiers:
                 project_name: functest
                 criteria: 90
                 blocking: false
-                clean_flag: false
                 description: >-
                     This test case runs the full suite of scenarios of the OpenStack
                     Rally suite using several threads and iterations.
@@ -546,7 +518,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     This test case deploys an OpenSource vIMS solution from Clearwater
                     using the Cloudify orchestrator. It also runs some signaling traffic.
@@ -563,7 +534,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                    Test suite from Parser project.
                 dependencies:
@@ -579,7 +549,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     VNF deployment with OpenBaton (Orchestra)
                 dependencies:
@@ -595,7 +564,6 @@ tiers:
                 project_name: opera
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     VNF deployment with OPEN-O
                 dependencies:
@@ -611,7 +579,6 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
-                clean_flag: true
                 description: >-
                     This test case is vRouter testing.
                 dependencies:
diff --git a/functest/ci/tier_builder.py b/functest/ci/tier_builder.py
index d03cd903..f8038468 100644
--- a/functest/ci/tier_builder.py
+++ b/functest/ci/tier_builder.py
@@ -52,7 +52,6 @@ class TierBuilder(object):
                                        dependency=dep,
                                        criteria=dic_testcase['criteria'],
                                        blocking=dic_testcase['blocking'],
-                                       clean_flag=dic_testcase['clean_flag'],
                                        description=dic_testcase['description'])
                 if (testcase.is_compatible(self.ci_installer,
                                            self.ci_scenario) and
diff --git a/functest/ci/tier_handler.py b/functest/ci/tier_handler.py
index 36ce245e..4f2f14ec 100644
--- a/functest/ci/tier_handler.py
+++ b/functest/ci/tier_handler.py
@@ -109,14 +109,12 @@ class TestCase(object):
                  dependency,
                  criteria,
                  blocking,
-                 clean_flag,
                  description=""):
         self.name = name
         self.enabled = enabled
         self.dependency = dependency
         self.criteria = criteria
         self.blocking = blocking
-        self.clean_flag = clean_flag
         self.description = description
 
     @staticmethod
@@ -149,9 +147,6 @@ class TestCase(object):
     def is_blocking(self):
         return self.blocking
 
-    def needs_clean(self):
-        return self.clean_flag
-
     def __str__(self):
         lines = split_text(self.description, LINE_LENGTH - 6)