diff options
| -rw-r--r-- | docker/Dockerfile.aarch64.patch | 33 | ||||
| -rw-r--r-- | qtip/collector/parser/grep.py | 2 | ||||
| -rw-r--r-- | qtip/driver/ansible_driver.py | 3 | ||||
| -rw-r--r-- | qtip/runner/runner.py | 4 | ||||
| -rwxr-xr-x | qtip/scripts/cleanup_creds.sh | 11 | ||||
| -rw-r--r-- | qtip/util/env.py | 55 | ||||
| -rw-r--r-- | tests/unit/util/env_test.py | 26 |
7 files changed, 83 insertions, 51 deletions
diff --git a/docker/Dockerfile.aarch64.patch b/docker/Dockerfile.aarch64.patch new file mode 100644 index 00000000..80529476 --- /dev/null +++ b/docker/Dockerfile.aarch64.patch @@ -0,0 +1,33 @@ +From: Cristina Pauna <cristina.pauna@enea.com> +Date: Mon, 13 Mar 2017 11:56:59 +0200 +Subject: [PATCH] Modify Dockerfile to build an aarch64 image + +This patch adapts the Dockerfile so that the qtip image +can be build on an aarch64 machine + +Signed-off-by: Cristina Pauna <cristina.pauna@enea.com> +--- + docker/Dockerfile | 14 ++++++-------- + 1 file changed, 6 insertions(+), 8 deletions(-) + +diff --git a/docker/Dockerfile b/docker/Dockerfile +index a4a7e47..5d4467b 100644 +--- a/docker/Dockerfile ++++ b/docker/Dockerfile +@@ -1,10 +1,10 @@ +-########################################## +-#####Docker container for QTIP############ +-########################################## ++################################################### ++##### AArch64 Docker container for QTIP############ ++################################################### + +-FROM ubuntu:16.04 +-MAINTAINER Yujun Zhang <zhang.yujunz@zte.com.cn> +-LABEL version="0.1" description="OPNFV QTIP Docker container" ++FROM aarch64/ubuntu:16.04 ++MAINTAINER Armband team <armband@enea.com> ++LABEL version="0.1" description="OPNFV QTIP AArch64 Docker container" + + ARG BRANCH=master + diff --git a/qtip/collector/parser/grep.py b/qtip/collector/parser/grep.py index 44edb5a1..d3a8210a 100644 --- a/qtip/collector/parser/grep.py +++ b/qtip/collector/parser/grep.py @@ -49,8 +49,6 @@ def _parse_logfile(config, paths): '{0}/{1}'.format(paths, regex_rules_by_file[GrepProp.FILENAME]) for regex in regex_rules_by_file['grep']: matches = grep_in_file(filename, regex) - for item in matches: - print item.groupdict() if len(matches) > 1: temp_dict = defaultdict(list) for item in [match.groupdict() for match in matches]: diff --git a/qtip/driver/ansible_driver.py b/qtip/driver/ansible_driver.py index 356c39b7..34ef46ed 100644 --- a/qtip/driver/ansible_driver.py +++ b/qtip/driver/ansible_driver.py @@ -46,6 +46,9 @@ class AnsibleDriver(object): self.env_setup_flag = True logger.info("Setup test enviroment, Done!") + def cleanup(self): + self.env.cleanup() + def run(self, metric_list, **kwargs): if 'args' in self.config: extra_vars = self.merge_two_dicts(kwargs, self.config['args']) diff --git a/qtip/runner/runner.py b/qtip/runner/runner.py index 47795bc3..8bdbfb78 100644 --- a/qtip/runner/runner.py +++ b/qtip/runner/runner.py @@ -36,7 +36,9 @@ def run_benchmark(result_dir, benchmarks): os.makedirs(result_dir) driver = AnsibleDriver({'args': {'result_dir': result_dir}}) driver.pre_run() - return driver.run(benchmarks) + result = driver.run(benchmarks) + driver.cleanup() + return result def generate_report(result_dir, start_time, stop_time): diff --git a/qtip/scripts/cleanup_creds.sh b/qtip/scripts/cleanup_creds.sh index b4eee924..1a7ddc1a 100755 --- a/qtip/scripts/cleanup_creds.sh +++ b/qtip/scripts/cleanup_creds.sh @@ -1,11 +1,20 @@ #! /bin/bash +############################################################################## +# Copyright (c) 2017 ZTE corp. and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## DEST_IP=$1 +PRIVATE_KEY=$2 HOSTNAME=$(hostname) sshoptions="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" case "$INSTALLER_TYPE" in fuel) - ssh $sshoptions -i ./config/QtipKey root@$DEST_IP "sed -i '/root@$HOSTNAME/d' /root/.ssh/authorized_keys" + ssh $sshoptions -i $PRIVATE_KEY root@$DEST_IP "sed -i '/root@$HOSTNAME/d' /root/.ssh/authorized_keys" ;; esac diff --git a/qtip/util/env.py b/qtip/util/env.py index 830c9b38..d00320c4 100644 --- a/qtip/util/env.py +++ b/qtip/util/env.py @@ -18,7 +18,7 @@ import paramiko from qtip.util.logger import QtipLogger -logger = QtipLogger('ansible_driver').get +logger = QtipLogger('env').get SCRIPT_DIR = path.join(path.dirname(__file__), path.pardir, 'scripts') KEYNAME = 'QtipKey' @@ -33,25 +33,25 @@ def all_files_exist(*files): flag = True for f_item in files: flag &= path.isfile(f_item) - print("Is {0} existed: {1}".format(f_item, flag)) + logger.info("Is {0} existed: {1}".format(f_item, flag)) return flag def clean_file(*files): if len(files) == 0: - print('Nothing to clean') + logger.info('Nothing to clean') return False def clean(f): try: if all_files_exist(f): os.remove(f) - print("Removed: {0}".format(f)) + logger.info("Removed: {0}".format(f)) else: - print("Not exists: {0}".format(f)) + logger.info("Not exists: {0}".format(f)) return True except OSError as error: - print("Not able to Remove: {0}".format(f), error) + logger.error("Not able to Remove: {0}".format(f), error) return False results = map(clean, files) @@ -78,7 +78,7 @@ class AnsibleEnvSetup(object): self.pass_keypair_to_remote() self.check_hosts_ssh_connectivity() except Exception as error: - print(error) + logger.info(error) sys.exit(1) def check_keypair(self, keypair): @@ -92,10 +92,9 @@ class AnsibleEnvSetup(object): def generate_default_keypair(self): if not all_files_exist(PRIVATE_KEY, PUBLIC_KEY): - print("Generate default keypair {0} under " - "{1}".format(KEYNAME, os.environ['HOME'])) - cmd = '''ssh-keygen -t rsa -N "" -f {0} -q -b 2048'''.format( - PRIVATE_KEY) + logger.info("Generate default keypair {0} under " + "{1}".format(KEYNAME, os.environ['HOME'])) + cmd = '''ssh-keygen -t rsa -N "" -f {0} -q -b 2048'''.format(PRIVATE_KEY) os.system(cmd) self.keypair['private'] = PRIVATE_KEY self.keypair['public'] = PUBLIC_KEY @@ -114,10 +113,10 @@ class AnsibleEnvSetup(object): time.sleep(2) ssh_cmd = '%s/qtip_creds.sh %s %s' % (SCRIPT_DIR, ip, private_key) os.system(ssh_cmd) - print('Pass keypair to remote hosts {0} successfully'.format(ip)) + logger.info('Pass keypair to remote hosts {0} successfully'.format(ip)) return True except Exception as error: - print(error) + logger.error(error) return False def check_hostfile(self, hostfile): @@ -132,8 +131,8 @@ class AnsibleEnvSetup(object): # check whether the file is already existed self.check_hostfile(HOST_FILE) except Exception: - print("Generate default hostfile {0} under " - "{1}".format(HOST_FILE, os.environ['HOME'])) + logger.info("Generate default hostfile {0} under " + "{1}".format(HOST_FILE, os.environ['HOME'])) self._generate_hostfile_via_installer() def _generate_hostfile_via_installer(self): @@ -156,11 +155,11 @@ class AnsibleEnvSetup(object): def fetch_host_ip_from_hostfile(self): self.host_ip_list = [] - print('Fetch host ips from hostfile...') + logger.info('Fetch host ips from hostfile...') with open(self.hostfile, 'r') as f: self.host_ip_list = re.findall('\d+.\d+.\d+.\d+', f.read()) if self.host_ip_list: - print("The remote compute nodes: {0}".format(self.host_ip_list)) + logger.info("The remote compute nodes: {0}".format(self.host_ip_list)) else: raise ValueError("The hostfile doesn't include host ip addresses.") @@ -172,7 +171,7 @@ class AnsibleEnvSetup(object): @staticmethod def _ssh_is_ok(ip, private_key, attempts=100): - print('Check hosts {0} ssh connectivity...'.format(ip)) + logger.info('Check hosts {0} ssh connectivity...'.format(ip)) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, key_filename=private_key) @@ -181,24 +180,28 @@ class AnsibleEnvSetup(object): try: stdin, stdout, stderr = ssh.exec_command('uname') if not stderr.readlines(): - print("{0}: SSH test successful.".format(ip)) + logger.info("{0}: SSH test successful.".format(ip)) return True except socket.error: - print("%s times ssh test......failed." % str(attempt + 1)) + logger.debug("%s times ssh test......failed." % str(attempt + 1)) if attempt == (attempts - 1): return False time.sleep(2) return False def cleanup(self): - IF_DEBUG = os.getenv('IF_DEBUG') + CI_DEBUG = os.getenv('CI_DEBUG') - if IF_DEBUG: + if CI_DEBUG: logger.info("DEBUG Mode: please do cleanup by manual.") else: - logger.info("Cleanup hostfile and keypair.") - clean_file(self.hostfile, self.keypair, self.keypair + '.pub') - for ip in self.host_ip_list: logger.info("Cleanup authorized_keys from {0}...".format(ip)) - os.system('bash %s/cleanup_creds.sh {0}'.format(ip)) + cmd = 'bash {0}/cleanup_creds.sh {1} {2}'.format( + SCRIPT_DIR, ip, self.keypair['private']) + os.system(cmd) + + logger.info("Cleanup hostfile and keypair.") + clean_file(self.hostfile, + self.keypair['private'], + self.keypair['public']) diff --git a/tests/unit/util/env_test.py b/tests/unit/util/env_test.py index 62d12a13..793d1e4e 100644 --- a/tests/unit/util/env_test.py +++ b/tests/unit/util/env_test.py @@ -72,12 +72,10 @@ def test_init(ansible_envsetup): assert ansible_envsetup.host_ip_list == [] -def test_setup_exception(capsys, mocker, ansible_envsetup, hostfile): +def test_setup_exception(mocker, ansible_envsetup, hostfile): with mock.patch.object(AnsibleEnvSetup, 'check_hostfile', side_effect=RuntimeError()): mock_os = mocker.patch('sys.exit') ansible_envsetup.setup({'hostfile': str(hostfile)}) - out, error = capsys.readouterr() - assert out == '\n' assert mock_os.call_count == 1 @@ -178,21 +176,17 @@ def test_pass_keypair_to_remote_failed(mocker, ansible_envsetup): assert "Failed on passing keypair to remote." in str(excinfo.value) -def test_pass_keypair(monkeypatch, capsys, mocker, ansible_envsetup): +def test_pass_keypair(monkeypatch, mocker, ansible_envsetup): monkeypatch.setattr(time, 'sleep', lambda s: None) mock_os = mocker.patch('os.system') ansible_envsetup._pass_keypair('10.20.0.3', str(private_key)) assert mock_os.call_count == 2 - out, error = capsys.readouterr() - assert "Pass keypair to remote hosts 10.20.0.3 successfully" in out -def test_pass_keypair_exception(capsys, ansible_envsetup): +def test_pass_keypair_exception(ansible_envsetup): with mock.patch('os.system', side_effect=Exception()) as mock_os: result = ansible_envsetup._pass_keypair('10.20.0.3', str(private_key)) assert result is False - out, error = capsys.readouterr() - assert out == '\n' assert mock_os.call_count == 1 @@ -304,20 +298,10 @@ def test_ssh_is_ok(mocker, ansible_envsetup, private_key, stderrinfo, expected): test_ssh_client.exec_command.assert_called_with('uname') -@pytest.mark.parametrize("attempts, expected", [ - (1, - 'Check hosts 10.20.0.3 ssh connectivity...\n1 times ssh test......failed.\n'), - (2, - 'Check hosts 10.20.0.3 ssh connectivity...\n' - '1 times ssh test......failed.\n' - '2 times ssh test......failed.\n') -]) -def test_ssh_exception(capsys, monkeypatch, mocker, ansible_envsetup, attempts, expected): +def test_ssh_exception(monkeypatch, mocker, ansible_envsetup): monkeypatch.setattr(time, 'sleep', lambda s: None) mock_sshclient = mocker.patch('paramiko.SSHClient') test_ssh_client = mock_sshclient.return_value test_ssh_client.exec_command.side_effect = socket.error() - result = ansible_envsetup._ssh_is_ok('10.20.0.3', str(private_key), attempts=attempts) - out, error = capsys.readouterr() - assert expected == out + result = ansible_envsetup._ssh_is_ok('10.20.0.3', str(private_key), attempts=1) assert result is False |