Define create_snapshot() and clean() in TestCase

They replace the former calls in run_tests which are not suitable for all
test cases. Now any test case can define how to clean its resources.

If the snapshot cannot be created, the test case is considered as
failed. Only a message is printed if any failure during cleaning.

It also defines a new class called OSGCTestCase useful for test cases
which don't clean their OpenStack resources.
All test cases located in opnfv_tests/openstack inherit from it to
keep the global behavior unchanged.

It also deletes exit instructions in openstack_clean and
openstack_snapshot, removes clean flags in testcases.yaml and updates
the related utils.

All Docs are modified as well.

JIRA: FUNCTEST-438

Change-Id: I8938e6255708012380389763a24059ace4ce45d8
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

4 files changed, 59 insertions, 19 deletions

diff --git a/functest/tests/unit/ci/test_run_tests.py b/functest/tests/unit/ci/test_run_tests.py
index 88e5d2b8..fb8cb391 100644
--- a/functest/tests/unit/ci/test_run_tests.py
+++ b/functest/tests/unit/ci/test_run_tests.py
@@ -38,8 +38,12 @@ class RunTestsTesting(unittest.TestCase):
                       'OS_PASSWORD': 'test_password'}
         self.test = {'test_name': 'test_name'}
         self.tier = mock.Mock()
+        test1 = mock.Mock()
+        test1.get_name.return_value = 'test1'
+        test2 = mock.Mock()
+        test2.get_name.return_value = 'test2'
         attrs = {'get_name.return_value': 'test_tier',
-                 'get_tests.return_value': ['test1', 'test2'],
+                 'get_tests.return_value': [test1, test2],
                  'get_ci_loop.return_value': 'test_ci_loop',
                  'get_test_names.return_value': ['test1', 'test2']}
         self.tier.configure_mock(**attrs)
@@ -70,16 +74,6 @@ class RunTestsTesting(unittest.TestCase):
                         return_value=self.creds):
             self.runner.source_rc_file()
 
-    @mock.patch('functest.ci.run_tests.os_snapshot.main')
-    def test_generate_os_snapshot(self, mock_os_snap):
-        self.runner.generate_os_snapshot()
-        self.assertTrue(mock_os_snap.called)
-
-    @mock.patch('functest.ci.run_tests.os_clean.main')
-    def test_cleanup(self, mock_os_clean):
-        self.runner.cleanup()
-        self.assertTrue(mock_os_clean.called)
-
     def test_get_run_dict_if_defined_default(self):
         mock_obj = mock.Mock()
         with mock.patch('functest.ci.run_tests.'
@@ -137,8 +131,6 @@ class RunTestsTesting(unittest.TestCase):
 
     @mock.patch('functest.ci.run_tests.Runner.print_separator')
     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
-    @mock.patch('functest.ci.run_tests.Runner.generate_os_snapshot')
-    @mock.patch('functest.ci.run_tests.Runner.cleanup')
     @mock.patch('importlib.import_module', name="module",
                 return_value=mock.Mock(test_class=mock.Mock(
                     side_effect=FakeModule)))
@@ -161,10 +153,10 @@ class RunTestsTesting(unittest.TestCase):
     def test_run_tier_default(self, mock_logger_info):
         with mock.patch('functest.ci.run_tests.Runner.print_separator'), \
                 mock.patch(
-                    'functest.ci.run_tests.Runner.run_test') as mock_method:
+                    'functest.ci.run_tests.Runner.run_test',
+                    return_value=TestCase.EX_OK) as mock_method:
             self.runner.run_tier(self.tier)
-            mock_method.assert_any_call('test1', 'test_tier')
-            mock_method.assert_any_call('test2', 'test_tier')
+            mock_method.assert_any_call(mock.ANY, 'test_tier')
             self.assertTrue(mock_logger_info.called)
 
     @mock.patch('functest.ci.run_tests.logger.info')
@@ -237,7 +229,8 @@ class RunTestsTesting(unittest.TestCase):
         with mock.patch('functest.ci.run_tests.tb.TierBuilder',
                         return_value=mock_obj), \
             mock.patch('functest.ci.run_tests.Runner.source_rc_file'), \
-                mock.patch('functest.ci.run_tests.Runner.run_test') as m:
+                mock.patch('functest.ci.run_tests.Runner.run_test',
+                           return_value=TestCase.EX_OK) as m:
             self.assertEqual(self.runner.main(**kwargs),
                              run_tests.Result.EX_OK)
             self.assertTrue(m.called)
diff --git a/functest/tests/unit/ci/test_tier_builder.py b/functest/tests/unit/ci/test_tier_builder.py
index 989c0870..ab75e15b 100644
--- a/functest/tests/unit/ci/test_tier_builder.py
+++ b/functest/tests/unit/ci/test_tier_builder.py
@@ -24,7 +24,6 @@ class TierBuilderTesting(unittest.TestCase):
                          'case_name': 'test_name',
                          'criteria': 'test_criteria',
                          'blocking': 'test_blocking',
-                         'clean_flag': 'test_clean_flag',
                          'description': 'test_desc'}
 
         self.dic_tier = {'name': 'test_tier',
diff --git a/functest/tests/unit/ci/test_tier_handler.py b/functest/tests/unit/ci/test_tier_handler.py
index c93fffd3..1909ac22 100644
--- a/functest/tests/unit/ci/test_tier_handler.py
+++ b/functest/tests/unit/ci/test_tier_handler.py
@@ -34,7 +34,6 @@ class TierHandlerTesting(unittest.TestCase):
                                               self.mock_depend,
                                               'test_criteria',
                                               'test_blocking',
-                                              'test_clean_flag',
                                               description='test_desc')
 
         self.dependency = tier_handler.Dependency('test_installer',
diff --git a/functest/tests/unit/core/test_testcase.py b/functest/tests/unit/core/test_testcase.py
index ef0983cc..0a6f0c9f 100644
--- a/functest/tests/unit/core/test_testcase.py
+++ b/functest/tests/unit/core/test_testcase.py
@@ -221,6 +221,55 @@ class TestCaseTesting(unittest.TestCase):
         self.assertIn(duration, message)
         self.assertIn('FAIL', message)
 
+    def test_create_snapshot(self):
+        self.assertEqual(self.test.create_snapshot(),
+                         testcase.TestCase.EX_OK)
+
+    def test_clean(self):
+        self.assertEqual(self.test.clean(), None)
+
+
+class OSGCTestCaseTesting(unittest.TestCase):
+    """The class testing OSGCTestCase."""
+    # pylint: disable=missing-docstring
+
+    def setUp(self):
+        self.test = testcase.OSGCTestCase()
+
+    @mock.patch('functest.utils.openstack_snapshot.main',
+                side_effect=Exception)
+    def test_create_snapshot_exc(self, mock_method=None):
+        self.assertEqual(self.test.create_snapshot(),
+                         testcase.TestCase.EX_RUN_ERROR)
+        mock_method.assert_called_once_with()
+
+    @mock.patch('functest.utils.openstack_snapshot.main', return_value=-1)
+    def test_create_snapshot_ko(self, mock_method=None):
+        self.assertEqual(self.test.create_snapshot(),
+                         testcase.TestCase.EX_RUN_ERROR)
+        mock_method.assert_called_once_with()
+
+    @mock.patch('functest.utils.openstack_snapshot.main', return_value=0)
+    def test_create_snapshot_env(self, mock_method=None):
+        self.assertEqual(self.test.create_snapshot(),
+                         testcase.TestCase.EX_OK)
+        mock_method.assert_called_once_with()
+
+    @mock.patch('functest.utils.openstack_clean.main', side_effect=Exception)
+    def test_clean_exc(self, mock_method=None):
+        self.assertEqual(self.test.clean(), None)
+        mock_method.assert_called_once_with()
+
+    @mock.patch('functest.utils.openstack_clean.main', return_value=-1)
+    def test_clean_ko(self, mock_method=None):
+        self.assertEqual(self.test.clean(), None)
+        mock_method.assert_called_once_with()
+
+    @mock.patch('functest.utils.openstack_clean.main', return_value=0)
+    def test_clean(self, mock_method=None):
+        self.assertEqual(self.test.clean(), None)
+        mock_method.assert_called_once_with()
+
 
 if __name__ == "__main__":
     logging.disable(logging.CRITICAL)