diff options
| -rw-r--r-- | functest/opnfv_tests/openstack/rally/rally.py | 93 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/tempest/conf_utils.py | 58 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/tempest/tempest.py | 18 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/rally/test_rally.py | 65 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/tempest/test_conf_utils.py | 22 | ||||
| -rw-r--r-- | functest/tests/unit/openstack/tempest/test_tempest.py | 25 |
6 files changed, 153 insertions, 128 deletions
diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index ecffd3f44..da2866a24 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -11,7 +11,9 @@ """Rally testcases implementation.""" from __future__ import division +from __future__ import print_function +import fileinput import json import logging import os @@ -28,7 +30,6 @@ from xtesting.core import testcase import yaml from functest.core import singlevm -from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils import config from functest.utils import env @@ -38,10 +39,13 @@ LOGGER = logging.getLogger(__name__) class RallyBase(singlevm.VmReady2): """Base class form Rally testcases implementation.""" - # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-instance-attributes, too-many-public-methods TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat', 'keystone', 'neutron', 'nova', 'quotas'] + RALLY_CONF_PATH = "/etc/rally/rally.conf" + RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename( + 'functest', 'ci/rally_aarch64_patch.conf') RALLY_DIR = pkg_resources.resource_filename( 'functest', 'opnfv_tests/openstack/rally') RALLY_SCENARIO_DIR = pkg_resources.resource_filename( @@ -146,6 +150,57 @@ class RallyBase(singlevm.VmReady2): return test_file_name @staticmethod + def get_verifier_deployment_id(): + """ + Returns deployment id for active Rally deployment + """ + cmd = ("rally deployment list | awk '/" + + getattr(config.CONF, 'rally_deployment_name') + + "/ {print $2}'") + proc = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + deployment_uuid = proc.stdout.readline().rstrip() + return deployment_uuid + + @staticmethod + def create_rally_deployment(environ=None): + """Create new rally deployment""" + # set the architecture to default + pod_arch = env.get("POD_ARCH") + arch_filter = ['aarch64'] + + if pod_arch and pod_arch in arch_filter: + LOGGER.info("Apply aarch64 specific to rally config...") + with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile: + rally_patch_conf = pfile.read() + + for line in fileinput.input(RallyBase.RALLY_CONF_PATH): + print(line, end=' ') + if "cirros|testvm" in line: + print(rally_patch_conf) + + LOGGER.info("Creating Rally environment...") + try: + cmd = ['rally', 'deployment', 'destroy', + '--deployment', + str(getattr(config.CONF, 'rally_deployment_name'))] + output = subprocess.check_output(cmd) + LOGGER.info("%s\n%s", " ".join(cmd), output) + except subprocess.CalledProcessError: + pass + + cmd = ['rally', 'deployment', 'create', '--fromenv', + '--name', str(getattr(config.CONF, 'rally_deployment_name'))] + output = subprocess.check_output(cmd, env=environ) + LOGGER.info("%s\n%s", " ".join(cmd), output) + + cmd = ['rally', 'deployment', 'check'] + output = subprocess.check_output(cmd) + LOGGER.info("%s\n%s", " ".join(cmd), output) + return RallyBase.get_verifier_deployment_id() + + @staticmethod def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'): """Set keystone_default_role in rally.conf""" if env.get("NEW_USER_ROLE").lower() != "member": @@ -516,8 +571,9 @@ class RallyBase(singlevm.VmReady2): 'nb success': success_rate}}) self.details = payload - def generate_html_report(self): - """Save all task reports as single HTML + @staticmethod + def export_task(file_name, export_type="html"): + """Export all task results (e.g. html or xunit report) Raises: subprocess.CalledProcessError: if Rally doesn't return 0 @@ -525,9 +581,26 @@ class RallyBase(singlevm.VmReady2): Returns: None """ - cmd = ["rally", "task", "report", "--deployment", + cmd = ["rally", "task", "export", "--type", export_type, + "--deployment", str(getattr(config.CONF, 'rally_deployment_name')), - "--out", "{}/{}.html".format(self.results_dir, self.case_name)] + "--to", file_name] + LOGGER.debug('running command: %s', cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + LOGGER.info("%s\n%s", " ".join(cmd), output) + + @staticmethod + def verify_report(file_name, uuid, export_type="html"): + """Generate the verifier report (e.g. html or xunit report) + + Raises: + subprocess.CalledProcessError: if Rally doesn't return 0 + + Returns: + None + """ + cmd = ["rally", "verify", "report", "--type", export_type, + "--uuid", uuid, "--to", file_name] LOGGER.debug('running command: %s', cmd) output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) LOGGER.info("%s\n%s", " ".join(cmd), output) @@ -564,11 +637,15 @@ class RallyBase(singlevm.VmReady2): del environ['OS_TENANT_ID'] except Exception: # pylint: disable=broad-except pass - conf_utils.create_rally_deployment(environ=environ) + self.create_rally_deployment(environ=environ) self.prepare_run(**kwargs) self.run_tests(**kwargs) self._generate_report() - self.generate_html_report() + self.export_task( + "{}/{}.html".format(self.results_dir, self.case_name)) + self.export_task( + "{}/{}.xml".format(self.results_dir, self.case_name), + export_type="junit-xml") res = testcase.TestCase.EX_OK except Exception as exc: # pylint: disable=broad-except LOGGER.error('Error with run: %s', exc) diff --git a/functest/opnfv_tests/openstack/tempest/conf_utils.py b/functest/opnfv_tests/openstack/tempest/conf_utils.py index 12671d4c1..d490ab036 100644 --- a/functest/opnfv_tests/openstack/tempest/conf_utils.py +++ b/functest/opnfv_tests/openstack/tempest/conf_utils.py @@ -10,11 +10,8 @@ """Tempest configuration utilities.""" -from __future__ import print_function - import json import logging -import fileinput import os import subprocess @@ -27,9 +24,6 @@ from functest.utils import env from functest.utils import functest_utils -RALLY_CONF_PATH = "/etc/rally/rally.conf" -RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename( - 'functest', 'ci/rally_aarch64_patch.conf') GLANCE_IMAGE_PATH = os.path.join( getattr(config.CONF, 'dir_functest_images'), getattr(config.CONF, 'openstack_image_file_name')) @@ -46,44 +40,6 @@ CI_INSTALLER_TYPE = env.get('INSTALLER_TYPE') LOGGER = logging.getLogger(__name__) -def create_rally_deployment(environ=None): - """Create new rally deployment""" - # set the architecture to default - pod_arch = env.get("POD_ARCH") - arch_filter = ['aarch64'] - - if pod_arch and pod_arch in arch_filter: - LOGGER.info("Apply aarch64 specific to rally config...") - with open(RALLY_AARCH64_PATCH_PATH, "r") as pfile: - rally_patch_conf = pfile.read() - - for line in fileinput.input(RALLY_CONF_PATH): - print(line, end=' ') - if "cirros|testvm" in line: - print(rally_patch_conf) - - LOGGER.info("Creating Rally environment...") - - try: - cmd = ['rally', 'deployment', 'destroy', - '--deployment', - str(getattr(config.CONF, 'rally_deployment_name'))] - output = subprocess.check_output(cmd) - LOGGER.info("%s\n%s", " ".join(cmd), output) - except subprocess.CalledProcessError: - pass - - cmd = ['rally', 'deployment', 'create', '--fromenv', - '--name', str(getattr(config.CONF, 'rally_deployment_name'))] - output = subprocess.check_output(cmd, env=environ) - LOGGER.info("%s\n%s", " ".join(cmd), output) - - cmd = ['rally', 'deployment', 'check'] - output = subprocess.check_output(cmd) - LOGGER.info("%s\n%s", " ".join(cmd), output) - return get_verifier_deployment_id() - - def create_verifier(): """Create new verifier""" LOGGER.info("Create verifier from existing repo...") @@ -119,20 +75,6 @@ def get_verifier_id(): return verifier_uuid -def get_verifier_deployment_id(): - """ - Returns deployment id for active Rally deployment - """ - cmd = ("rally deployment list | awk '/" + - getattr(config.CONF, 'rally_deployment_name') + - "/ {print $2}'") - proc = subprocess.Popen(cmd, shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - deployment_uuid = proc.stdout.readline().rstrip() - return deployment_uuid - - def get_verifier_repo_dir(verifier_id): """ Returns installed verifier repo directory for Tempest diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index 011c8b6fb..1ae37f91c 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -24,6 +24,7 @@ from xtesting.core import testcase import yaml from functest.core import singlevm +from functest.opnfv_tests.openstack.rally import rally from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils import config from functest.utils import env @@ -289,14 +290,6 @@ class TempestCommon(singlevm.VmReady2): LOGGER.info("Tempest %s success_rate is %s%%", self.case_name, self.result) - def generate_report(self): - """Generate verification report.""" - html_file = os.path.join(self.res_dir, - "tempest-report.html") - cmd = ["rally", "verify", "report", "--type", "html", "--uuid", - self.verification_id, "--to", str(html_file)] - subprocess.check_output(cmd) - def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'): """Set image name as tempest img_name_regex""" rconfig = configparser.RawConfigParser() @@ -420,7 +413,7 @@ class TempestCommon(singlevm.VmReady2): del environ['OS_TENANT_ID'] except Exception: # pylint: disable=broad-except pass - self.deployment_id = conf_utils.create_rally_deployment( + self.deployment_id = rally.RallyBase.create_rally_deployment( environ=environ) if not self.deployment_id: raise Exception("Deployment create failed") @@ -471,7 +464,12 @@ class TempestCommon(singlevm.VmReady2): self.apply_tempest_blacklist() self.run_verifier_tests(**kwargs) self.parse_verifier_result() - self.generate_report() + rally.RallyBase.verify_report( + os.path.join(self.res_dir, "tempest-report.html"), + self.verification_id) + rally.RallyBase.verify_report( + os.path.join(self.res_dir, "tempest-report.xml"), + self.verification_id, "junit-xml") res = testcase.TestCase.EX_OK except Exception: # pylint: disable=broad-except LOGGER.exception('Error with run') diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index 0a605dd4c..67cf07e79 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -69,6 +69,21 @@ class OSRallyTesting(unittest.TestCase): return True return False + @mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'RallyBase.get_verifier_deployment_id', return_value='foo') + @mock.patch('subprocess.check_output') + def test_create_rally_deployment(self, mock_exec, mock_get_id): + # pylint: disable=unused-argument + self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo') + calls = [ + mock.call(['rally', 'deployment', 'destroy', '--deployment', + str(getattr(config.CONF, 'rally_deployment_name'))]), + mock.call(['rally', 'deployment', 'create', '--fromenv', '--name', + str(getattr(config.CONF, 'rally_deployment_name'))], + env=None), + mock.call(['rally', 'deployment', 'check'])] + mock_exec.assert_has_calls(calls) + @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists') @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' @@ -314,7 +329,7 @@ class OSRallyTesting(unittest.TestCase): self.rally_base.clean() self.assertEqual(mock_delete_flavor.call_count, 1) - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'create_rally_deployment') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'prepare_run') @@ -323,19 +338,19 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' '_generate_report') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' - 'generate_html_report') + 'export_task') def test_run_default(self, *args): self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK) for func in args: func.assert_called() - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'create_rally_deployment', side_effect=Exception) def test_run_exception_create_rally_dep(self, mock_create_rally_dep): self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR) mock_create_rally_dep.assert_called() - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'create_rally_deployment', return_value=mock.Mock()) @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'prepare_run', side_effect=Exception) @@ -372,22 +387,44 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('subprocess.check_output', side_effect=subprocess.CalledProcessError('', '')) - def test_generate_html_report_ko(self, *args): + def test_export_task_ko(self, *args): + file_name = "{}/{}.html".format( + self.rally_base.results_dir, self.rally_base.case_name) with self.assertRaises(subprocess.CalledProcessError): - self.rally_base.generate_html_report() - cmd = ["rally", "task", "report", "--deployment", + self.rally_base.export_task(file_name) + cmd = ["rally", "task", "export", "--type", "html", "--deployment", str(getattr(config.CONF, 'rally_deployment_name')), - "--out", "{}/{}.html".format( - self.rally_base.results_dir, self.rally_base.case_name)] + "--to", file_name] args[0].assert_called_with(cmd, stderr=subprocess.STDOUT) @mock.patch('subprocess.check_output', return_value=None) - def test_generate_html_report(self, *args): - self.assertEqual(self.rally_base.generate_html_report(), None) - cmd = ["rally", "task", "report", "--deployment", + def test_export_task(self, *args): + file_name = "{}/{}.html".format( + self.rally_base.results_dir, self.rally_base.case_name) + self.assertEqual(self.rally_base.export_task(file_name), None) + cmd = ["rally", "task", "export", "--type", "html", "--deployment", str(getattr(config.CONF, 'rally_deployment_name')), - "--out", "{}/{}.html".format( - self.rally_base.results_dir, self.rally_base.case_name)] + "--to", file_name] + args[0].assert_called_with(cmd, stderr=subprocess.STDOUT) + + @mock.patch('subprocess.check_output', + side_effect=subprocess.CalledProcessError('', '')) + def test_verify_report_ko(self, *args): + file_name = "{}/{}.html".format( + self.rally_base.results_dir, self.rally_base.case_name) + with self.assertRaises(subprocess.CalledProcessError): + self.rally_base.verify_report(file_name, "1") + cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1", + "--to", file_name] + args[0].assert_called_with(cmd, stderr=subprocess.STDOUT) + + @mock.patch('subprocess.check_output', return_value=None) + def test_verify_report(self, *args): + file_name = "{}/{}.html".format( + self.rally_base.results_dir, self.rally_base.case_name) + self.assertEqual(self.rally_base.verify_report(file_name, "1"), None) + cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1", + "--to", file_name] args[0].assert_called_with(cmd, stderr=subprocess.STDOUT) diff --git a/functest/tests/unit/openstack/tempest/test_conf_utils.py b/functest/tests/unit/openstack/tempest/test_conf_utils.py index 19b07ba3e..f99b02b50 100644 --- a/functest/tests/unit/openstack/tempest/test_conf_utils.py +++ b/functest/tests/unit/openstack/tempest/test_conf_utils.py @@ -13,6 +13,7 @@ import unittest import mock +from functest.opnfv_tests.openstack.rally import rally from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils import config @@ -21,21 +22,6 @@ class OSTempestConfUtilsTesting(unittest.TestCase): # pylint: disable=too-many-public-methods @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils' - '.get_verifier_deployment_id', return_value='foo') - @mock.patch('subprocess.check_output') - def test_create_rally_deployment(self, mock_exec, mock_get_id): - # pylint: disable=unused-argument - self.assertEqual(conf_utils.create_rally_deployment(), 'foo') - calls = [ - mock.call(['rally', 'deployment', 'destroy', '--deployment', - str(getattr(config.CONF, 'rally_deployment_name'))]), - mock.call(['rally', 'deployment', 'create', '--fromenv', '--name', - str(getattr(config.CONF, 'rally_deployment_name'))], - env=None), - mock.call(['rally', 'deployment', 'check'])] - mock_exec.assert_has_calls(calls) - - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils' '.LOGGER.debug') def test_create_verifier(self, mock_logger_debug): mock_popen = mock.Mock() @@ -52,8 +38,8 @@ class OSTempestConfUtilsTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' 'create_verifier', return_value=mock.Mock()) - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' - 'create_rally_deployment', return_value=mock.Mock()) + @mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'RallyBase.create_rally_deployment', return_value=mock.Mock()) def test_get_verifier_id_default(self, mock_rally, mock_tempest): # pylint: disable=unused-argument setattr(config.CONF, 'tempest_verifier_name', 'test_verifier_name') @@ -76,7 +62,7 @@ class OSTempestConfUtilsTesting(unittest.TestCase): mock_stdout.configure_mock(**attrs) mock_popen.return_value = mock_stdout - self.assertEqual(conf_utils.get_verifier_deployment_id(), + self.assertEqual(rally.RallyBase.get_verifier_deployment_id(), 'test_deploy_id') def test_get_verif_repo_dir_default(self): diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py index c1f245c72..646f88290 100644 --- a/functest/tests/unit/openstack/tempest/test_tempest.py +++ b/functest/tests/unit/openstack/tempest/test_tempest.py @@ -16,7 +16,6 @@ from xtesting.core import testcase from functest.opnfv_tests.openstack.tempest import tempest from functest.opnfv_tests.openstack.tempest import conf_utils -from functest.utils import config class OSTempestTesting(unittest.TestCase): @@ -26,15 +25,15 @@ class OSTempestTesting(unittest.TestCase): with mock.patch('os_client_config.get_config'), \ mock.patch('shade.OpenStackCloud'), \ mock.patch('functest.core.tenantnetwork.NewProject'), \ - mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' - 'conf_utils.create_rally_deployment'), \ + mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'RallyBase.create_rally_deployment'), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'conf_utils.create_verifier'), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'conf_utils.get_verifier_id', return_value='test_deploy_id'), \ - mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' - 'conf_utils.get_verifier_deployment_id', + mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'RallyBase.get_verifier_deployment_id', return_value='test_deploy_id'), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'conf_utils.get_verifier_repo_dir', @@ -171,19 +170,6 @@ class OSTempestTesting(unittest.TestCase): mock_logger_info. \ assert_any_call("Starting Tempest test suite: '%s'.", cmd) - @mock.patch('subprocess.check_output') - def test_generate_report(self, mock_popen): - self.tempestcommon.verification_id = "1234" - html_file = os.path.join( - os.path.join( - getattr(config.CONF, 'dir_results'), - self.tempestcommon.case_name), - "tempest-report.html") - cmd = ["rally", "verify", "report", "--type", "html", "--uuid", - "1234", "--to", html_file] - self.tempestcommon.generate_report() - mock_popen.assert_called_once_with(cmd) - @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'os.path.exists', return_value=False) @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.os.makedirs', @@ -260,8 +246,7 @@ class OSTempestTesting(unittest.TestCase): 'apply_tempest_blacklist'), \ mock.patch.object(self.tempestcommon, 'run_verifier_tests'), \ mock.patch.object(self.tempestcommon, - 'parse_verifier_result'), \ - mock.patch.object(self.tempestcommon, 'generate_report'): + 'parse_verifier_result'): self._test_run(testcase.TestCase.EX_OK) args[0].assert_called_once_with() |