Generate xunit reports (rally and tempest)

It adds xunit reports for rally-based and tempest-based testcases.
It completes the reports provided by snaps (thanks to Xtesting).

All rally related operations are moved to rally.
It allows removing the rally dependency to tempest which was false.

Change-Id: Ia7d2476f58f4f68b7c88442e50cad844037a36e9
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

3 files changed, 60 insertions, 52 deletions

diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py
index 0a605dd4c..67cf07e79 100644
--- a/functest/tests/unit/openstack/rally/test_rally.py
+++ b/functest/tests/unit/openstack/rally/test_rally.py
@@ -69,6 +69,21 @@ class OSRallyTesting(unittest.TestCase):
             return True
         return False
 
+    @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+                'RallyBase.get_verifier_deployment_id', return_value='foo')
+    @mock.patch('subprocess.check_output')
+    def test_create_rally_deployment(self, mock_exec, mock_get_id):
+        # pylint: disable=unused-argument
+        self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo')
+        calls = [
+            mock.call(['rally', 'deployment', 'destroy', '--deployment',
+                       str(getattr(config.CONF, 'rally_deployment_name'))]),
+            mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
+                       str(getattr(config.CONF, 'rally_deployment_name'))],
+                      env=None),
+            mock.call(['rally', 'deployment', 'check'])]
+        mock_exec.assert_has_calls(calls)
+
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
@@ -314,7 +329,7 @@ class OSRallyTesting(unittest.TestCase):
             self.rally_base.clean()
             self.assertEqual(mock_delete_flavor.call_count, 1)
 
-    @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+    @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
                 'create_rally_deployment')
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
                 'prepare_run')
@@ -323,19 +338,19 @@ class OSRallyTesting(unittest.TestCase):
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
                 '_generate_report')
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
-                'generate_html_report')
+                'export_task')
     def test_run_default(self, *args):
         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
         for func in args:
             func.assert_called()
 
-    @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+    @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
                 'create_rally_deployment', side_effect=Exception)
     def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
         mock_create_rally_dep.assert_called()
 
-    @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+    @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
                 'create_rally_deployment', return_value=mock.Mock())
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
                 'prepare_run', side_effect=Exception)
@@ -372,22 +387,44 @@ class OSRallyTesting(unittest.TestCase):
 
     @mock.patch('subprocess.check_output',
                 side_effect=subprocess.CalledProcessError('', ''))
-    def test_generate_html_report_ko(self, *args):
+    def test_export_task_ko(self, *args):
+        file_name = "{}/{}.html".format(
+            self.rally_base.results_dir, self.rally_base.case_name)
         with self.assertRaises(subprocess.CalledProcessError):
-            self.rally_base.generate_html_report()
-        cmd = ["rally", "task", "report", "--deployment",
+            self.rally_base.export_task(file_name)
+        cmd = ["rally", "task", "export", "--type", "html", "--deployment",
                str(getattr(config.CONF, 'rally_deployment_name')),
-               "--out", "{}/{}.html".format(
-                   self.rally_base.results_dir, self.rally_base.case_name)]
+               "--to", file_name]
         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 
     @mock.patch('subprocess.check_output', return_value=None)
-    def test_generate_html_report(self, *args):
-        self.assertEqual(self.rally_base.generate_html_report(), None)
-        cmd = ["rally", "task", "report", "--deployment",
+    def test_export_task(self, *args):
+        file_name = "{}/{}.html".format(
+            self.rally_base.results_dir, self.rally_base.case_name)
+        self.assertEqual(self.rally_base.export_task(file_name), None)
+        cmd = ["rally", "task", "export", "--type", "html", "--deployment",
                str(getattr(config.CONF, 'rally_deployment_name')),
-               "--out", "{}/{}.html".format(
-                   self.rally_base.results_dir, self.rally_base.case_name)]
+               "--to", file_name]
+        args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
+    @mock.patch('subprocess.check_output',
+                side_effect=subprocess.CalledProcessError('', ''))
+    def test_verify_report_ko(self, *args):
+        file_name = "{}/{}.html".format(
+            self.rally_base.results_dir, self.rally_base.case_name)
+        with self.assertRaises(subprocess.CalledProcessError):
+            self.rally_base.verify_report(file_name, "1")
+        cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
+               "--to", file_name]
+        args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
+    @mock.patch('subprocess.check_output', return_value=None)
+    def test_verify_report(self, *args):
+        file_name = "{}/{}.html".format(
+            self.rally_base.results_dir, self.rally_base.case_name)
+        self.assertEqual(self.rally_base.verify_report(file_name, "1"), None)
+        cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
+               "--to", file_name]
         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 
 
diff --git a/functest/tests/unit/openstack/tempest/test_conf_utils.py b/functest/tests/unit/openstack/tempest/test_conf_utils.py
index 19b07ba3e..f99b02b50 100644
--- a/functest/tests/unit/openstack/tempest/test_conf_utils.py
+++ b/functest/tests/unit/openstack/tempest/test_conf_utils.py
@@ -13,6 +13,7 @@ import unittest
 
 import mock
 
+from functest.opnfv_tests.openstack.rally import rally
 from functest.opnfv_tests.openstack.tempest import conf_utils
 from functest.utils import config
 
@@ -21,21 +22,6 @@ class OSTempestConfUtilsTesting(unittest.TestCase):
     # pylint: disable=too-many-public-methods
 
     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils'
-                '.get_verifier_deployment_id', return_value='foo')
-    @mock.patch('subprocess.check_output')
-    def test_create_rally_deployment(self, mock_exec, mock_get_id):
-        # pylint: disable=unused-argument
-        self.assertEqual(conf_utils.create_rally_deployment(), 'foo')
-        calls = [
-            mock.call(['rally', 'deployment', 'destroy', '--deployment',
-                       str(getattr(config.CONF, 'rally_deployment_name'))]),
-            mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
-                       str(getattr(config.CONF, 'rally_deployment_name'))],
-                      env=None),
-            mock.call(['rally', 'deployment', 'check'])]
-        mock_exec.assert_has_calls(calls)
-
-    @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils'
                 '.LOGGER.debug')
     def test_create_verifier(self, mock_logger_debug):
         mock_popen = mock.Mock()
@@ -52,8 +38,8 @@ class OSTempestConfUtilsTesting(unittest.TestCase):
 
     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
                 'create_verifier', return_value=mock.Mock())
-    @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
-                'create_rally_deployment', return_value=mock.Mock())
+    @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+                'RallyBase.create_rally_deployment', return_value=mock.Mock())
     def test_get_verifier_id_default(self, mock_rally, mock_tempest):
         # pylint: disable=unused-argument
         setattr(config.CONF, 'tempest_verifier_name', 'test_verifier_name')
@@ -76,7 +62,7 @@ class OSTempestConfUtilsTesting(unittest.TestCase):
             mock_stdout.configure_mock(**attrs)
             mock_popen.return_value = mock_stdout
 
-            self.assertEqual(conf_utils.get_verifier_deployment_id(),
+            self.assertEqual(rally.RallyBase.get_verifier_deployment_id(),
                              'test_deploy_id')
 
     def test_get_verif_repo_dir_default(self):
diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py
index c1f245c72..646f88290 100644
--- a/functest/tests/unit/openstack/tempest/test_tempest.py
+++ b/functest/tests/unit/openstack/tempest/test_tempest.py
@@ -16,7 +16,6 @@ from xtesting.core import testcase
 
 from functest.opnfv_tests.openstack.tempest import tempest
 from functest.opnfv_tests.openstack.tempest import conf_utils
-from functest.utils import config
 
 
 class OSTempestTesting(unittest.TestCase):
@@ -26,15 +25,15 @@ class OSTempestTesting(unittest.TestCase):
         with mock.patch('os_client_config.get_config'), \
                 mock.patch('shade.OpenStackCloud'), \
                 mock.patch('functest.core.tenantnetwork.NewProject'), \
-                mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
-                           'conf_utils.create_rally_deployment'), \
+                mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+                           'RallyBase.create_rally_deployment'), \
                 mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
                            'conf_utils.create_verifier'), \
                 mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
                            'conf_utils.get_verifier_id',
                            return_value='test_deploy_id'), \
-                mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
-                           'conf_utils.get_verifier_deployment_id',
+                mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+                           'RallyBase.get_verifier_deployment_id',
                            return_value='test_deploy_id'), \
                 mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
                            'conf_utils.get_verifier_repo_dir',
@@ -171,19 +170,6 @@ class OSTempestTesting(unittest.TestCase):
                 mock_logger_info. \
                     assert_any_call("Starting Tempest test suite: '%s'.", cmd)
 
-    @mock.patch('subprocess.check_output')
-    def test_generate_report(self, mock_popen):
-        self.tempestcommon.verification_id = "1234"
-        html_file = os.path.join(
-            os.path.join(
-                getattr(config.CONF, 'dir_results'),
-                self.tempestcommon.case_name),
-            "tempest-report.html")
-        cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
-               "1234", "--to", html_file]
-        self.tempestcommon.generate_report()
-        mock_popen.assert_called_once_with(cmd)
-
     @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
                 'os.path.exists', return_value=False)
     @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.os.makedirs',
@@ -260,8 +246,7 @@ class OSTempestTesting(unittest.TestCase):
                                   'apply_tempest_blacklist'), \
                 mock.patch.object(self.tempestcommon, 'run_verifier_tests'), \
                 mock.patch.object(self.tempestcommon,
-                                  'parse_verifier_result'), \
-                mock.patch.object(self.tempestcommon, 'generate_report'):
+                                  'parse_verifier_result'):
             self._test_run(testcase.TestCase.EX_OK)
             args[0].assert_called_once_with()