From 2a0702ea914a574ffa7c6d6f83a5c606e24ebd0a Mon Sep 17 00:00:00 2001 From: Cédric Ollivier Date: Tue, 7 May 2019 11:02:43 +0200 Subject: Update to Python3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Functest containers leverage on Python3 instead of python2. https://mail.python.org/pipermail/python-dev/2018-March/152348.html It also updates robotframework librairies to latest release and Vmtp to master ([1] is needed) It patches cloudify rest client to support python3. Vmtp is currently disabled because it currently supports python2 only. [1] https://github.com/openstack/vmtp/commit/a5d062881d91bf4f547d92c6e289bea30feb5d6e#diff-b4ef698db8ca845e5845c4618278f29a Change-Id: I39964a212ec2d4dbf53c9ea7c63f02e0b6a05b48 Signed-off-by: Cédric Ollivier --- functest/ci/testcases.yaml | 1 + .../opnfv_tests/openstack/barbican/barbican.py | 2 +- functest/opnfv_tests/openstack/patrole/patrole.py | 2 +- functest/opnfv_tests/openstack/rally/rally.py | 12 ++-- .../opnfv_tests/openstack/refstack/refstack.py | 2 +- functest/opnfv_tests/openstack/tempest/tempest.py | 28 ++++---- functest/opnfv_tests/vnf/epc/juju_epc.py | 3 +- .../vnf/router/vnf_controller/ssh_client.py | 2 +- functest/tests/unit/openstack/rally/test_rally.py | 4 +- .../tests/unit/openstack/tempest/test_tempest.py | 4 +- functest/tests/unit/utils/test_functest_utils.py | 80 +++++++--------------- functest/utils/functest_utils.py | 4 +- 12 files changed, 58 insertions(+), 86 deletions(-) (limited to 'functest') diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index f88265178..ee66426fd 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -382,6 +382,7 @@ tiers: - case_name: vmtp project_name: functest + enabled: false criteria: 100 blocking: false description: >- diff --git a/functest/opnfv_tests/openstack/barbican/barbican.py b/functest/opnfv_tests/openstack/barbican/barbican.py index b9488c2b6..7b1bb24f7 100644 --- a/functest/opnfv_tests/openstack/barbican/barbican.py +++ b/functest/opnfv_tests/openstack/barbican/barbican.py @@ -36,6 +36,6 @@ class Barbican(tempest.TempestCommon): if not rconfig.has_section('image-feature-enabled'): rconfig.add_section('image-feature-enabled') rconfig.set('image-feature-enabled', 'api_v1', False) - with open(self.conf_file, 'wb') as config_file: + with open(self.conf_file, 'w') as config_file: rconfig.write(config_file) self.backup_tempest_config(self.conf_file, self.res_dir) diff --git a/functest/opnfv_tests/openstack/patrole/patrole.py b/functest/opnfv_tests/openstack/patrole/patrole.py index 45378b6cd..9bd877b60 100644 --- a/functest/opnfv_tests/openstack/patrole/patrole.py +++ b/functest/opnfv_tests/openstack/patrole/patrole.py @@ -27,7 +27,7 @@ class Patrole(tempest.TempestCommon): rconfig.add_section('rbac') rconfig.set('rbac', 'enable_rbac', True) rconfig.set('rbac', 'rbac_test_role', kwargs.get('role', 'admin')) - with open(self.conf_file, 'wb') as config_file: + with open(self.conf_file, 'w') as config_file: rconfig.write(config_file) self.backup_tempest_config(self.conf_file, self.res_dir) diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 25de3cfb6..674c74252 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -166,7 +166,7 @@ class RallyBase(singlevm.VmReady2): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) deployment_uuid = proc.stdout.readline().rstrip() - return deployment_uuid + return deployment_uuid.decode() @staticmethod def create_rally_deployment(environ=None): @@ -215,7 +215,7 @@ class RallyBase(singlevm.VmReady2): rconfig.add_section('openstack') rconfig.set( 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE")) - with open(rally_conf, 'wb') as config_file: + with open(rally_conf, 'w') as config_file: rconfig.write(config_file) @staticmethod @@ -226,7 +226,7 @@ class RallyBase(singlevm.VmReady2): rconfig.read(rally_conf) if rconfig.has_option('openstack', 'keystone_default_role'): rconfig.remove_option('openstack', 'keystone_default_role') - with open(rally_conf, 'wb') as config_file: + with open(rally_conf, 'w') as config_file: rconfig.write(config_file) @staticmethod @@ -240,7 +240,7 @@ class RallyBase(singlevm.VmReady2): taskid_re = re.compile('^Task +(.*): started$') for line in cmd_raw.splitlines(True): line = line.strip() - match = taskid_re.match(line) + match = taskid_re.match(line.decode()) if match: return match.group(1) return None @@ -657,8 +657,8 @@ class RallyBase(singlevm.VmReady2): "{}/{}.xml".format(self.results_dir, self.case_name), export_type="junit-xml") res = testcase.TestCase.EX_OK - except Exception as exc: # pylint: disable=broad-except - LOGGER.error('Error with run: %s', exc) + except Exception: # pylint: disable=broad-except + LOGGER.exception('Error with run:') self.result = 0 res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() diff --git a/functest/opnfv_tests/openstack/refstack/refstack.py b/functest/opnfv_tests/openstack/refstack/refstack.py index 814cb255b..9eb937165 100644 --- a/functest/opnfv_tests/openstack/refstack/refstack.py +++ b/functest/opnfv_tests/openstack/refstack/refstack.py @@ -55,7 +55,7 @@ class Refstack(tempest.TempestCommon): yaml_data2 = "" for line in output.splitlines(): try: - grp = re.search(r'^([^\[]*)(\[.*\])\n*$', line) + grp = re.search(r'^([^\[]*)(\[.*\])\n*$', line.decode()) yaml_data2 = "{}\n{}: {}".format( yaml_data2, grp.group(1), grp.group(2)) except Exception: # pylint: disable=broad-except diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index 0874e6d61..30e9ce60b 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -143,7 +143,7 @@ class TempestCommon(singlevm.VmReady2): stderr=subprocess.STDOUT) for line in proc.stdout: LOGGER.info(line.rstrip()) - new_line = line.replace(' ', '').split('|') + new_line = line.decode().replace(' ', '').split('|') if 'Tests' in new_line: break if 'Testscount' in new_line: @@ -199,7 +199,7 @@ class TempestCommon(singlevm.VmReady2): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) verifier_uuid = proc.stdout.readline().rstrip() - return verifier_uuid + return verifier_uuid.decode() @staticmethod def get_verifier_repo_dir(verifier_id): @@ -235,7 +235,7 @@ class TempestCommon(singlevm.VmReady2): for key, value in sub_conf.items(): rconfig.set(section, key, value) - with open(conf_file, 'wb') as config_file: + with open(conf_file, 'w') as config_file: rconfig.write(config_file) @staticmethod @@ -403,12 +403,12 @@ class TempestCommon(singlevm.VmReady2): with proc.stdout: for line in iter(proc.stdout.readline, b''): - if re.search(r"\} tempest\.", line): + if re.search(r"\} tempest\.", line.decode()): LOGGER.info(line.rstrip()) - elif re.search(r'(?=\(UUID=(.*)\))', line): + elif re.search(r'(?=\(UUID=(.*)\))', line.decode()): self.verification_id = re.search( - r'(?=\(UUID=(.*)\))', line).group(1) - f_stdout.write(line) + r'(?=\(UUID=(.*)\))', line.decode()).group(1) + f_stdout.write(line.decode()) proc.wait() f_stdout.close() @@ -473,7 +473,7 @@ class TempestCommon(singlevm.VmReady2): rconfig.add_section('openstack') rconfig.set('openstack', 'img_name_regex', '^{}$'.format( self.image.name)) - with open(rally_conf, 'wb') as config_file: + with open(rally_conf, 'w') as config_file: rconfig.write(config_file) def update_default_role(self, rally_conf='/etc/rally/rally.conf'): @@ -486,7 +486,7 @@ class TempestCommon(singlevm.VmReady2): if not rconfig.has_section('openstack'): rconfig.add_section('openstack') rconfig.set('openstack', 'swift_operator_role', role.name) - with open(rally_conf, 'wb') as config_file: + with open(rally_conf, 'w') as config_file: rconfig.write(config_file) def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'): @@ -499,7 +499,7 @@ class TempestCommon(singlevm.VmReady2): rconfig.set('DEFAULT', 'use_stderr', False) rconfig.set('DEFAULT', 'log-file', 'rally.log') rconfig.set('DEFAULT', 'log_dir', self.res_dir) - with open(rally_conf, 'wb') as config_file: + with open(rally_conf, 'w') as config_file: rconfig.write(config_file) @staticmethod @@ -519,7 +519,7 @@ class TempestCommon(singlevm.VmReady2): rconfig.remove_option('DEFAULT', 'log-file') if rconfig.has_option('DEFAULT', 'log_dir'): rconfig.remove_option('DEFAULT', 'log_dir') - with open(rally_conf, 'wb') as config_file: + with open(rally_conf, 'w') as config_file: rconfig.write(config_file) def update_network_section(self): @@ -530,7 +530,7 @@ class TempestCommon(singlevm.VmReady2): rconfig.add_section('network') rconfig.set('network', 'public_network_id', self.ext_net.id) rconfig.set('network', 'floating_network_name', self.ext_net.name) - with open(self.conf_file, 'wb') as config_file: + with open(self.conf_file, 'w') as config_file: rconfig.write(config_file) def update_compute_section(self): @@ -540,7 +540,7 @@ class TempestCommon(singlevm.VmReady2): if not rconfig.has_section('compute'): rconfig.add_section('compute') rconfig.set('compute', 'fixed_network_name', self.network.name) - with open(self.conf_file, 'wb') as config_file: + with open(self.conf_file, 'w') as config_file: rconfig.write(config_file) def update_scenario_section(self): @@ -567,7 +567,7 @@ class TempestCommon(singlevm.VmReady2): rconfig.set( 'scenario', 'img_properties', functest_utils.convert_dict_to_ini(extra_properties)) - with open(self.conf_file, 'wb') as config_file: + with open(self.conf_file, 'w') as config_file: rconfig.write(config_file) def configure(self, **kwargs): # pylint: disable=unused-argument diff --git a/functest/opnfv_tests/vnf/epc/juju_epc.py b/functest/opnfv_tests/vnf/epc/juju_epc.py index cbf531b6b..93961ada9 100644 --- a/functest/opnfv_tests/vnf/epc/juju_epc.py +++ b/functest/opnfv_tests/vnf/epc/juju_epc.py @@ -273,7 +273,8 @@ class JujuEpc(singlevm.VmReady2): for i in range(10): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) self.__logger.info("%s\n%s", " ".join(cmd), output) - ret = re.search(r'(?=workload:({})\))'.format(status), output) + ret = re.search( + r'(?=workload:({})\))'.format(status), output.decode()) if ret: self.__logger.info("%s workload is %s", name, status) break diff --git a/functest/opnfv_tests/vnf/router/vnf_controller/ssh_client.py b/functest/opnfv_tests/vnf/router/vnf_controller/ssh_client.py index c5f554cbd..67f1e0d6d 100644 --- a/functest/opnfv_tests/vnf/router/vnf_controller/ssh_client.py +++ b/functest/opnfv_tests/vnf/router/vnf_controller/ssh_client.py @@ -110,7 +110,7 @@ class SshClient(object): # pylint: disable=too-many-instance-attributes cmd) break - res_buff += res + res_buff += res.decode() self.logger.debug("Response : '%s'", res_buff) return res_buff diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index 5dc38a20f..1b790a0bf 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -101,12 +101,12 @@ class OSRallyTesting(unittest.TestCase): mock_os_makedirs.assert_called() def test_get_task_id_default(self): - cmd_raw = 'Task 1: started' + cmd_raw = b'Task 1: started' self.assertEqual(self.rally_base.get_task_id(cmd_raw), '1') def test_get_task_id_missing_id(self): - cmd_raw = '' + cmd_raw = b'' self.assertEqual(self.rally_base.get_task_id(cmd_raw), None) diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py index 119959670..a500a37e1 100644 --- a/functest/tests/unit/openstack/tempest/test_tempest.py +++ b/functest/tests/unit/openstack/tempest/test_tempest.py @@ -271,7 +271,7 @@ class OSTempestTesting(unittest.TestCase): with mock.patch('functest.opnfv_tests.openstack.tempest.' 'tempest.subprocess.Popen') as mock_popen: mock_stdout = mock.Mock() - attrs = {'stdout.readline.return_value': 'test_deploy_id'} + attrs = {'stdout.readline.return_value': b'test_deploy_id'} mock_stdout.configure_mock(**attrs) mock_popen.return_value = mock_stdout @@ -283,7 +283,7 @@ class OSTempestTesting(unittest.TestCase): with mock.patch('functest.opnfv_tests.openstack.tempest.' 'tempest.subprocess.Popen') as mock_popen: mock_stdout = mock.Mock() - attrs = {'stdout.readline.return_value': 'test_deploy_id'} + attrs = {'stdout.readline.return_value': b'test_deploy_id'} mock_stdout.configure_mock(**attrs) mock_popen.return_value = mock_stdout diff --git a/functest/tests/unit/utils/test_functest_utils.py b/functest/tests/unit/utils/test_functest_utils.py index da3b03c80..ba022e377 100644 --- a/functest/tests/unit/utils/test_functest_utils.py +++ b/functest/tests/unit/utils/test_functest_utils.py @@ -15,6 +15,7 @@ import unittest import mock import pkg_resources +import six from functest.utils import functest_utils @@ -97,23 +98,15 @@ class FunctestUtilsTesting(unittest.TestCase): as mock_subproc_open, \ mock.patch('six.moves.builtins.open', mock.mock_open()) as mopen: - - FunctestUtilsTesting.readline = 0 - - mock_obj = mock.Mock() - attrs = {'readline.side_effect': self.cmd_readline()} - mock_obj.configure_mock(**attrs) - + stream = six.BytesIO() + stream.write(self.cmd_readline().encode()) mock_obj2 = mock.Mock() - attrs = {'stdout': mock_obj, 'wait.return_value': 1} + attrs = {'stdout': stream, 'wait.return_value': 1} mock_obj2.configure_mock(**attrs) - mock_subproc_open.return_value = mock_obj2 - - resp = functest_utils.execute_command(self.cmd, info=True, - error_msg=self.error_msg, - verbose=True, - output_file=self.output_file) + resp = functest_utils.execute_command( + self.cmd, info=True, error_msg=self.error_msg, verbose=True, + output_file=self.output_file) self.assertEqual(resp, 1) msg_exec = ("Executing command: '%s'" % self.cmd) mock_logger_info.assert_called_once_with(msg_exec) @@ -126,23 +119,15 @@ class FunctestUtilsTesting(unittest.TestCase): as mock_subproc_open, \ mock.patch('six.moves.builtins.open', mock.mock_open()) as mopen: - - FunctestUtilsTesting.readline = 0 - - mock_obj = mock.Mock() - attrs = {'readline.side_effect': self.cmd_readline()} - mock_obj.configure_mock(**attrs) - + stream = six.BytesIO() + stream.write(self.cmd_readline().encode()) mock_obj2 = mock.Mock() - attrs = {'stdout': mock_obj, 'wait.return_value': 0} + attrs = {'stdout': stream, 'wait.return_value': 0} mock_obj2.configure_mock(**attrs) - mock_subproc_open.return_value = mock_obj2 - - resp = functest_utils.execute_command(self.cmd, info=True, - error_msg=self.error_msg, - verbose=True, - output_file=self.output_file) + resp = functest_utils.execute_command( + self.cmd, info=True, error_msg=self.error_msg, verbose=True, + output_file=self.output_file) self.assertEqual(resp, 0) msg_exec = ("Executing command: '%s'" % self.cmd) mock_logger_info.assert_called_once_with(msg_exec) @@ -153,23 +138,15 @@ class FunctestUtilsTesting(unittest.TestCase): # pylint: disable=unused-argument with mock.patch('functest.utils.functest_utils.subprocess.Popen') \ as mock_subproc_open: - - FunctestUtilsTesting.readline = 2 - - mock_obj = mock.Mock() - attrs = {'readline.side_effect': self.cmd_readline()} - mock_obj.configure_mock(**attrs) - + stream = six.BytesIO() + stream.write(self.cmd_readline().encode()) mock_obj2 = mock.Mock() - attrs = {'stdout': mock_obj, 'wait.return_value': 0} + attrs = {'stdout': stream, 'wait.return_value': 0} mock_obj2.configure_mock(**attrs) - mock_subproc_open.return_value = mock_obj2 - - resp = functest_utils.execute_command(self.cmd, info=False, - error_msg="", - verbose=False, - output_file=None) + resp = functest_utils.execute_command( + self.cmd, info=False, error_msg="", verbose=False, + output_file=None) self.assertEqual(resp, 0) @mock.patch('sys.stdout') @@ -177,22 +154,15 @@ class FunctestUtilsTesting(unittest.TestCase): # pylint: disable=unused-argument with mock.patch('functest.utils.functest_utils.subprocess.Popen') \ as mock_subproc_open: - - FunctestUtilsTesting.readline = 2 - mock_obj = mock.Mock() - attrs = {'readline.side_effect': self.cmd_readline()} - mock_obj.configure_mock(**attrs) - + stream = six.BytesIO() + stream.write(self.cmd_readline().encode()) mock_obj2 = mock.Mock() - attrs = {'stdout': mock_obj, 'wait.return_value': 1} + attrs = {'stdout': stream, 'wait.return_value': 1} mock_obj2.configure_mock(**attrs) - mock_subproc_open.return_value = mock_obj2 - - resp = functest_utils.execute_command(self.cmd, info=False, - error_msg="", - verbose=False, - output_file=None) + resp = functest_utils.execute_command( + self.cmd, info=False, error_msg="", verbose=False, + output_file=None) self.assertEqual(resp, 1) def test_get_param_from_yaml_failed(self): diff --git a/functest/utils/functest_utils.py b/functest/utils/functest_utils.py index 19d5baf92..4bc417d61 100644 --- a/functest/utils/functest_utils.py +++ b/functest/utils/functest_utils.py @@ -43,9 +43,9 @@ def execute_command(cmd, info=False, error_msg="", ofd = open(output_file, "w") for line in iter(popen.stdout.readline, b''): if output_file: - ofd.write(line) + ofd.write(line.decode()) else: - line = line.replace('\n', '') + line = line.decode().replace('\n', '') print(line) sys.stdout.flush() if output_file: -- cgit 1.2.3-korg