diff options
| -rw-r--r-- | jjb/compass4nfv/compass-ci-jobs.yml | 2 | ||||
| -rw-r--r-- | jjb/compass4nfv/compass-project-jobs.yml | 2 | ||||
| -rw-r--r-- | jjb/qtip/qtip-ci-jobs.yml | 10 | ||||
| -rw-r--r-- | jjb/releng-macros.yaml | 2 | ||||
| -rw-r--r-- | jjb/vswitchperf/vswitchperf.yml | 6 | ||||
| -rw-r--r-- | jjb/yardstick/yardstick-ci-jobs.yml | 16 | ||||
| -rw-r--r-- | utils/installer-adapter/ApexAdapter.py | 5 | ||||
| -rw-r--r-- | utils/installer-adapter/CompassAdapter.py | 5 | ||||
| -rw-r--r-- | utils/installer-adapter/FuelAdapter.py | 71 | ||||
| -rw-r--r-- | utils/installer-adapter/JoidAdapter.py | 5 | ||||
| -rw-r--r-- | utils/installer-adapter/RelengLogger.py | 1 | ||||
| -rw-r--r-- | utils/installer-adapter/SSHUtils.py | 216 |
12 files changed, 174 insertions, 167 deletions
diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index da882cdfe..eb9113114 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -100,7 +100,7 @@ blocking-jobs: - 'compass-os-.*?-{pod}-daily-.*?' - 'compass-os-.*?-baremetal-daily-.*?' - - 'compass-verify-[^-]*' + - 'compass-verify-[^-]*-[^-]*' block-level: 'NODE' wrappers: diff --git a/jjb/compass4nfv/compass-project-jobs.yml b/jjb/compass4nfv/compass-project-jobs.yml index 4d799af8e..5ce906482 100644 --- a/jjb/compass4nfv/compass-project-jobs.yml +++ b/jjb/compass4nfv/compass-project-jobs.yml @@ -56,7 +56,7 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - 'compass-verify-[^-]*' + - 'compass-verify-[^-]*-[^-]*' - 'compass-os-.*?-virtual-daily-.*?' block-level: 'NODE' diff --git a/jjb/qtip/qtip-ci-jobs.yml b/jjb/qtip/qtip-ci-jobs.yml index d0d6b47a3..cca8cee4c 100644 --- a/jjb/qtip/qtip-ci-jobs.yml +++ b/jjb/qtip/qtip-ci-jobs.yml @@ -40,6 +40,10 @@ installer: fuel auto-trigger-name: 'qtip-daily-zte-pod2-trigger' <<: *master + - zte-pod3: + installer: fuel + auto-trigger-name: 'qtip-daily-zte-pod3-trigger' + <<: *master #-------------------------------- jobs: @@ -120,5 +124,9 @@ - trigger: name: 'qtip-daily-zte-pod2-trigger' triggers: - - timed: '0 5 * * *' + - timed: '0 7 * * *' +- trigger: + name: 'qtip-daily-zte-pod3-trigger' + triggers: + - timed: '0 1 * * *' diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index d2dc1d1ec..2ebd7757b 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -176,7 +176,7 @@ mkdir -p upload mv docs_output "$local_path" - gsutil -m cp -r "$local_path" "gs://$GS_URL" + gsutil -m cp -r "$local_path" "gs://$gs_path" gsutil -m setmeta \ -h "Content-Type:text/html" \ diff --git a/jjb/vswitchperf/vswitchperf.yml b/jjb/vswitchperf/vswitchperf.yml index 233cf08bc..3f7f6bf2d 100644 --- a/jjb/vswitchperf/vswitchperf.yml +++ b/jjb/vswitchperf/vswitchperf.yml @@ -14,10 +14,12 @@ branch: '{stream}' gs-pathname: '' disabled: false + slave-label: 'opnfv-build-ubuntu' - colorado: branch: 'stable/{stream}' gs-pathname: '/{stream}' disabled: false + slave-label: 'intel-pod3' - job-template: @@ -72,7 +74,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' + - '{slave-label}-defaults' scm: - gerrit-trigger-scm: @@ -134,7 +136,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' + - '{slave-label}-defaults' scm: - gerrit-trigger-scm: diff --git a/jjb/yardstick/yardstick-ci-jobs.yml b/jjb/yardstick/yardstick-ci-jobs.yml index c10daabe0..9d80e42ae 100644 --- a/jjb/yardstick/yardstick-ci-jobs.yml +++ b/jjb/yardstick/yardstick-ci-jobs.yml @@ -315,7 +315,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: name: 'yardstick-params-armband-baremetal' @@ -329,7 +329,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: name: 'yardstick-params-joid-baremetal' @@ -343,14 +343,14 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: name: 'yardstick-params-intel-pod8' parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: name: 'yardstick-params-lf-pod1' @@ -445,7 +445,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -453,7 +453,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -461,7 +461,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -469,7 +469,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' ####################### diff --git a/utils/installer-adapter/ApexAdapter.py b/utils/installer-adapter/ApexAdapter.py index bf451f3d2..17a27b10a 100644 --- a/utils/installer-adapter/ApexAdapter.py +++ b/utils/installer-adapter/ApexAdapter.py @@ -8,9 +8,6 @@ ############################################################################## -from SSHUtils import SSH_Connection - - class ApexAdapter: def __init__(self, installer_ip): @@ -32,4 +29,4 @@ class ApexAdapter: pass def get_file_from_controller(self, origin, target, ip=None, options=None): - pass
\ No newline at end of file + pass diff --git a/utils/installer-adapter/CompassAdapter.py b/utils/installer-adapter/CompassAdapter.py index b40a8d788..47cbc646d 100644 --- a/utils/installer-adapter/CompassAdapter.py +++ b/utils/installer-adapter/CompassAdapter.py @@ -8,9 +8,6 @@ ############################################################################## -from SSHUtils import SSH_Connection - - class CompassAdapter: def __init__(self, installer_ip): @@ -32,4 +29,4 @@ class CompassAdapter: pass def get_file_from_controller(self, origin, target, ip=None, options=None): - pass
\ No newline at end of file + pass diff --git a/utils/installer-adapter/FuelAdapter.py b/utils/installer-adapter/FuelAdapter.py index 15f0e929f..672fd5175 100644 --- a/utils/installer-adapter/FuelAdapter.py +++ b/utils/installer-adapter/FuelAdapter.py @@ -1,14 +1,14 @@ ############################################################################## # Copyright (c) 2016 Ericsson AB and others. # Author: Jose Lausuch (jose.lausuch@ericsson.com) +# George Paraskevopoulos (geopar@intracom-telecom.com) # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - -from SSHUtils import SSH_Connection +import SSHUtils as ssh_utils import RelengLogger as rl @@ -16,25 +16,30 @@ class FuelAdapter: def __init__(self, installer_ip, user="root", password="r00tme"): self.installer_ip = installer_ip - self.user = user - self.password = password - self.connection = SSH_Connection( - installer_ip, self.user, self.password, use_system_keys=False) + self.installer_user = user + self.installer_password = password + self.installer_connection = ssh_utils.get_ssh_client( + installer_ip, + self.installer_user, + password=self.installer_password) self.logger = rl.Logger("Handler").getLogger() - def runcmd_fuel_nodes(self): - output, error = self.connection.run_remote_cmd('fuel nodes') + def runcmd_fuel_installer(self, cmd): + _, stdout, stderr = (self + .installer_connection + .exec_command(cmd)) + error = stderr.readlines() if len(error) > 0: - self.logger.error("error %s" % error) + self.logger.error("error %s" % ''.join(error)) return error + output = ''.join(stdout.readlines()) return output + def runcmd_fuel_nodes(self): + return self.runcmd_fuel_installer('fuel nodes') + def runcmd_fuel_env(self): - output, error = self.connection.run_remote_cmd('fuel env') - if len(error) > 0: - self.logger.error("error %s" % error) - return error - return output + return self.runcmd_fuel_installer('fuel env') def get_clusters(self): environments = [] @@ -183,8 +188,11 @@ class FuelAdapter: def get_file_from_installer(self, remote_path, local_path, options=None): self.logger.debug("Fetching %s from %s" % (remote_path, self.installer_ip)) - if self.connection.scp_get(local_path, remote_path) != 0: - self.logger.error("SCP failed to retrieve the file.") + get_file_result = ssh_utils.get_file(self.installer_connection, + remote_path, + local_path) + if get_file_result is None: + self.logger.error("SFTP failed to retrieve the file.") return 1 self.logger.info("%s successfully copied from Fuel to %s" % (remote_path, local_path)) @@ -193,6 +201,7 @@ class FuelAdapter: remote_path, local_path, ip=None, + user='root', options=None): if ip is None: controllers = self.get_controller_ips(options=options) @@ -204,16 +213,24 @@ class FuelAdapter: else: target_ip = ip - fuel_dir = '/root/scp/' - cmd = 'mkdir -p %s;rsync -Rav %s:%s %s' % ( - fuel_dir, target_ip, remote_path, fuel_dir) - self.logger.info("Copying %s from %s to Fuel..." % - (remote_path, target_ip)) - output, error = self.connection.run_remote_cmd(cmd) - self.logger.debug("Copying files from Fuel to %s..." % local_path) - self.get_file_from_installer( - fuel_dir + remote_path, local_path, options) - cmd = 'rm -r %s' % fuel_dir - output, error = self.connection.run_remote_cmd(cmd) + installer_jumphost = { + 'ip': self.installer_ip, + 'username': self.installer_user, + 'password': self.installer_password + } + controller_conn = ssh_utils.get_ssh_client( + target_ip, + user, + jumphost=installer_jumphost) + + self.logger.debug("Fetching %s from %s" % + (remote_path, target_ip)) + + get_file_result = ssh_utils.get_file(controller_conn, + remote_path, + local_path) + if get_file_result is None: + self.logger.error("SFTP failed to retrieve the file.") + return 1 self.logger.info("%s successfully copied from %s to %s" % (remote_path, target_ip, local_path)) diff --git a/utils/installer-adapter/JoidAdapter.py b/utils/installer-adapter/JoidAdapter.py index e78ca0fae..be8c2ebac 100644 --- a/utils/installer-adapter/JoidAdapter.py +++ b/utils/installer-adapter/JoidAdapter.py @@ -8,9 +8,6 @@ ############################################################################## -from SSHUtils import SSH_Connection - - class JoidAdapter: def __init__(self, installer_ip): @@ -32,4 +29,4 @@ class JoidAdapter: pass def get_file_from_controller(self, origin, target, ip=None, options=None): - pass
\ No newline at end of file + pass diff --git a/utils/installer-adapter/RelengLogger.py b/utils/installer-adapter/RelengLogger.py index b38e78095..6fa4ef2e2 100644 --- a/utils/installer-adapter/RelengLogger.py +++ b/utils/installer-adapter/RelengLogger.py @@ -22,7 +22,6 @@ # logger.debug("message to be shown with - DEBUG -") import logging -import os class Logger: diff --git a/utils/installer-adapter/SSHUtils.py b/utils/installer-adapter/SSHUtils.py index 9c92a3be1..c93888694 100644 --- a/utils/installer-adapter/SSHUtils.py +++ b/utils/installer-adapter/SSHUtils.py @@ -1,6 +1,7 @@ ############################################################################## # Copyright (c) 2015 Ericsson AB and others. -# Author: Jose Lausuch (jose.lausuch@ericsson.com) +# Authors: George Paraskevopoulos (geopar@intracom-telecom.com) +# Jose Lausuch (jose.lausuch@ericsson.com) # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -9,122 +10,111 @@ import paramiko -from scp import SCPClient -import time import RelengLogger as rl +import os +logger = rl.Logger('SSHUtils').getLogger() -class SSH_Connection: - - def __init__(self, - host, - user, - password, - use_system_keys=True, - private_key=None, - use_proxy=False, - proxy_host=None, - proxy_user=None, - proxy_password=None, - timeout=10): - self.host = host - self.user = user - self.password = password - self.use_system_keys = use_system_keys - self.private_key = private_key - self.use_proxy = use_proxy - self.proxy_host = proxy_host - self.proxy_user = proxy_user - self.proxy_password = proxy_password - self.timeout = timeout - paramiko.util.log_to_file("paramiko.log") - self.logger = rl.Logger("SSHUtils").getLogger() - - def connect(self): - client = paramiko.SSHClient() - if self.use_system_keys: - client.load_system_host_keys() - elif self.private_key: - client.load_host_keys(self.private_key) + +def get_ssh_client(hostname, username, password=None, jumphost=None): + client = None + try: + if jumphost is None: + client = paramiko.SSHClient() else: - client.load_host_keys('/dev/null') + client = JumpHostHopClient() + client.configure_jump_host(jumphost['ip'], + jumphost['username'], + jumphost['password']) + + if client is None: + raise Exception('Could not connect to client') client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect(hostname, + username=username, + password=password) + return client + except Exception, e: + logger.error(e) + return None - t = self.timeout - proxy = None - if self.use_proxy: - proxy_command = 'ssh -o UserKnownHostsFile=/dev/null ' - '-o StrictHostKeyChecking=no %s@%s -W %s:%s' % (self.proxy_user, - self.proxy_host, - self.host, 22) - proxy = paramiko.ProxyCommand(proxy_command) - self.logger.debug("Proxy command: %s" % proxy_command) - while t > 0: - try: - self.logger.debug( - "Trying to stablish ssh connection to %s..." % self.host) - client.connect(self.host, - username=self.user, - password=self.password, - look_for_keys=True, - sock=proxy, - pkey=self.private_key, - timeout=self.timeout) - self.logger.debug("Successfully connected to %s!" % self.host) - return client - except: - time.sleep(1) - t -= 1 - - if t == 0: - return None - - def scp_put(self, local_path, remote_path): - client = self.connect() - if client: - scp = SCPClient(client.get_transport()) - try: - scp.put(local_path, remote_path) - client.close() - return 0 - except Exception, e: - self.logger.error(e) - client.close() - return 1 - else: - self.logger.error("Cannot stablish ssh connection.") - - def scp_get(self, local_path, remote_path): - client = self.connect() - if client: - scp = SCPClient(client.get_transport()) - try: - scp.get(remote_path, local_path) - client.close() - return 0 - except Exception, e: - self.logger.error(e) - client.close() - return 1 - else: - self.logger.error("Cannot stablish ssh connection.") - return 1 - - def run_remote_cmd(self, command): - client = self.connect() - if client: - try: - stdin, stdout, stderr = client.exec_command(command) - out = '' - for line in stdout.readlines(): - out += line - err = stderr.readlines() - client.close() - return out, err - except: - client.close() - return 1 - else: - self.logger.error("Cannot stablish ssh connection.") - return 1 + +def get_file(ssh_conn, src, dest): + try: + sftp = ssh_conn.open_sftp() + sftp.get(src, dest) + return True + except Exception, e: + logger.error("Error [get_file(ssh_conn, '%s', '%s']: %s" % + (src, dest, e)) + return None + + +def put_file(ssh_conn, src, dest): + try: + sftp = ssh_conn.open_sftp() + sftp.put(src, dest) + return True + except Exception, e: + logger.error("Error [put_file(ssh_conn, '%s', '%s']: %s" % + (src, dest, e)) + return None + + +class JumpHostHopClient(paramiko.SSHClient): + ''' + Connect to a remote server using a jumphost hop + ''' + def __init__(self, *args, **kwargs): + self.logger = rl.Logger("JumpHostHopClient").getLogger() + self.jumphost_ssh = None + self.jumphost_transport = None + self.jumphost_channel = None + self.jumphost_ip = None + self.jumphost_ssh_key = None + self.local_ssh_key = os.path.join(os.getcwd(), 'id_rsa') + super(JumpHostHopClient, self).__init__(*args, **kwargs) + + def configure_jump_host(self, jh_ip, jh_user, jh_pass, + jh_ssh_key='/root/.ssh/id_rsa'): + self.jumphost_ip = jh_ip + self.jumphost_ssh_key = jh_ssh_key + self.jumphost_ssh = paramiko.SSHClient() + self.jumphost_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.jumphost_ssh.connect(jh_ip, + username=jh_user, + password=jh_pass) + self.jumphost_transport = self.jumphost_ssh.get_transport() + + def connect(self, hostname, port=22, username='root', password=None, + pkey=None, key_filename=None, timeout=None, allow_agent=True, + look_for_keys=True, compress=False, sock=None, gss_auth=False, + gss_kex=False, gss_deleg_creds=True, gss_host=None, + banner_timeout=None): + try: + if self.jumphost_ssh is None: + raise Exception('You must configure the jump ' + 'host before calling connect') + + get_file_res = get_file(self.jumphost_ssh, + self.jumphost_ssh_key, + self.local_ssh_key) + if get_file_res is None: + raise Exception('Could\'t fetch SSH key from jump host') + jumphost_key = (paramiko.RSAKey + .from_private_key_file(self.local_ssh_key)) + + self.jumphost_channel = self.jumphost_transport.open_channel( + "direct-tcpip", + (hostname, 22), + (self.jumphost_ip, 22)) + + self.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + super(JumpHostHopClient, self).connect(hostname, + username=username, + pkey=jumphost_key, + sock=self.jumphost_channel) + os.remove(self.local_ssh_key) + except Exception, e: + self.logger.error(e) |