Using paramiko for all ssh & scp

By using paramiko, we can now also use differnt credentials to login into
the PROX instances. In the ssh section of the environment file, you can
now also add the password parameter.

Change-Id: I08d31cbf0d02d7e82b7fbe34268851f73ff5dde0
Signed-off-by: Luc Provoost <luc.provoost@intel.com>

6 files changed, 118 insertions, 118 deletions

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
index 40375c54..2df78cbe 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
@@ -26,31 +26,28 @@ import time
 import subprocess
 import socket
 from rapid_log import RapidLog
+from rapid_sshclient import SSHClient
 
 class prox_ctrl(object):
-    def __init__(self, ip, key=None, user=None):
+    def __init__(self, ip, key=None, user=None, password = None):
         self._ip   = ip
         self._key  = key
         self._user = user
+        self._password = password
         self._proxsock = []
+        self._sshclient = SSHClient(ip = ip, user = user, password = password,
+                rsa_private_key = key)
 
     def ip(self):
         return self._ip
 
-    def test_connect(self):
-        """Simply try to run 'true' over ssh on remote system.
-        On failure, raise RuntimeWarning exception when possibly worth
-        retrying, and raise RuntimeError exception otherwise.
-        """
-        return self.run_cmd('test -e /opt/rapid/system_ready_for_rapid', True)
-
-    def connect(self):
+    def test_connection(self):
         attempts = 1
         RapidLog.debug("Trying to connect to machine \
                 on %s, attempt: %d" % (self._ip, attempts))
         while True:
             try:
-                self.test_connect()
+                self.run_cmd('test -e /opt/rapid/system_ready_for_rapid')
                 break
             except RuntimeWarning as ex:
                 RapidLog.debug("RuntimeWarning %d:\n%s"
@@ -87,18 +84,8 @@ class prox_ctrl(object):
         for sock in self._proxsock:
             sock.quit()
 
-    def run_cmd(self, command, _connect=False):
-        """Execute command over ssh on remote system.
-        Wait for remote command completion.
-        Return command output (combined stdout and stderr).
-        _connect argument is reserved for connect() method.
-        """
-        cmd = self._build_ssh(command)
-        try:
-            return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        except subprocess.CalledProcessError as ex:
-            RapidLog.exception('ssh returned exit status %d:\n%s'
-                    % (ex.returncode, ex.output.strip()))
+    def run_cmd(self, command):
+        self._sshclient.run_cmd(command)
 
     def prox_sock(self, port=8474):
         """Connect to the PROX instance on remote system.
@@ -114,64 +101,10 @@ class prox_ctrl(object):
             return None
 
     def scp_put(self, src, dst):
-        """Copy src file from local system to dst on remote system."""
-        cmd = [ 'scp',
-                '-B',
-                '-oStrictHostKeyChecking=no',
-                '-oUserKnownHostsFile=/dev/null',
-                '-oLogLevel=ERROR' ]
-        if self._key is not None:
-            cmd.extend(['-i', self._key])
-        cmd.append(src)
-        remote = ''
-        if self._user is not None:
-            remote += self._user + '@'
-        remote += self._ip + ':' + dst
-        cmd.append(remote)
-        try:
-            # Actually ignore output on success, but capture stderr on failure
-            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        except subprocess.CalledProcessError as ex:
-            RapidLog.exception('scp returned exit status %d:\n%s'
-                    % (ex.returncode, ex.output.strip()))
+        self._sshclient.scp_put(src, dst)
 
     def scp_get(self, src, dst):
-        """Copy src file from remote system to dst on local system."""
-        cmd = [ 'scp',
-                '-B',
-                '-oStrictHostKeyChecking=no',
-                '-oUserKnownHostsFile=/dev/null',
-                '-oLogLevel=ERROR' ]
-        if self._key is not None:
-            cmd.extend(['-i', self._key])
-        remote = ''
-        if self._user is not None:
-            remote += self._user + '@'
-        remote += self._ip + ':/home/' + self._user + src
-        cmd.append(remote)
-        cmd.append(dst)
-        try:
-            # Actually ignore output on success, but capture stderr on failure
-            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        except subprocess.CalledProcessError as ex:
-            RapidLog.exception('scp returned exit status %d:\n%s'
-                    % (ex.returncode, ex.output.strip()))
-
-    def _build_ssh(self, command):
-        cmd = [ 'ssh',
-                '-oBatchMode=yes',
-                '-oStrictHostKeyChecking=no',
-                '-oUserKnownHostsFile=/dev/null',
-                '-oLogLevel=ERROR' ]
-        if self._key is not None:
-            cmd.extend(['-i', self._key])
-        remote = ''
-        if self._user is not None:
-            remote += self._user + '@'
-        remote += self._ip
-        cmd.append(remote)
-        cmd.append(command)
-        return cmd
+        self._sshclient.scp_get('/home/' + self._user + src, dst)
 
 class prox_sock(object):
     def __init__(self, sock):
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
index f4c89ceb..e52b17db 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
@@ -17,7 +17,6 @@
 ##
 
 from rapid_log import RapidLog 
-from prox_ctrl import prox_ctrl
 from rapid_machine import RapidMachine
 from math import ceil, log2
 
@@ -49,8 +48,8 @@ class RapidGeneratorMachine(RapidMachine):
     """
     Class to deal with a generator PROX instance (VM, bare metal, container)
     """
-    def __init__(self, key, user, vim, rundir, resultsdir, machine_params,
-            configonly, ipv6):
+    def __init__(self, key, user, password, vim, rundir, resultsdir,
+            machine_params, configonly, ipv6):
         mac_address_size = 6
         ethertype_size = 2
         FCS_size = 4
@@ -78,8 +77,8 @@ class RapidGeneratorMachine(RapidMachine):
             self.bucket_size_exp = machine_params['bucket_size_exp']
         else:
             self.bucket_size_exp = 11
-        super().__init__(key, user, vim, rundir, resultsdir, machine_params,
-                configonly)
+        super().__init__(key, user, password, vim, rundir, resultsdir,
+                machine_params, configonly)
 
     def get_cores(self):
         return (self.machine_params['gencores'] +
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py
index 515bea55..c4b7247f 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py
@@ -25,12 +25,13 @@ class RapidMachine(object):
     """
     Class to deal with a PROX instance (VM, bare metal, container)
     """
-    def __init__(self, key, user, vim, rundir, resultsdir, machine_params,
-            configonly):
+    def __init__(self, key, user, password, vim, rundir, resultsdir,
+            machine_params, configonly):
         self.name = machine_params['name']
         self.ip = machine_params['admin_ip']
         self.key = key
         self.user = user
+        self.password = password
         self.rundir = rundir
         self.resultsdir = resultsdir
         self.dp_ports = []
@@ -58,11 +59,6 @@ class RapidMachine(object):
             PROXConfigfile.close()
             self.all_tasks_for_this_cfg = set(re.findall("task\s*=\s*(\d+)",PROXConfig))
 
-    def __del__(self):
-        if ((not self.configonly) and self.machine_params['prox_socket']):
-            self._client.scp_get('/prox.log', '{}/{}.prox.log'.format(
-                self.resultsdir, self.name))
-
     def get_cores(self):
         return (self.machine_params['cores'])
 
@@ -190,26 +186,36 @@ class RapidMachine(object):
 
     def start_prox(self, autostart=''):
         if self.machine_params['prox_socket']:
-            self._client = prox_ctrl(self.ip, self.key, self.user)
-            self._client.connect()
+            self._client = prox_ctrl(self.ip, self.key, self.user,
+                    self.password)
+            self._client.test_connection()
             if self.vim in ['OpenStack']:
                 self.devbind()
             if self.vim in ['kubernetes']:
                 self.read_cpuset()
                 self.read_cpuset_mems()
                 self.remap_all_cpus()
-            _, prox_config_file_name = os.path.split(self.machine_params['config_file'])
+            _, prox_config_file_name = os.path.split(self.
+                    machine_params['config_file'])
             self.generate_lua()
-            self._client.scp_put(self.machine_params['config_file'], '{}/{}'.format(self.rundir, prox_config_file_name))
-            if ((not self.configonly) and self.machine_params['prox_launch_exit']):
-                cmd = 'sudo {}/prox {} -t -o cli -f {}/{}'.format(self.rundir, autostart, self.rundir, prox_config_file_name)
-                RapidLog.debug("Starting PROX on {}: {}".format(self.name, cmd))
-                result = self._client.run_cmd(cmd, 'PROX Testing on {}'.format(self.name))
-                RapidLog.debug("Finished PROX on {}: {}".format(self.name, cmd))
+            self._client.scp_put(self.machine_params['config_file'], '{}/{}'.
+                    format(self.rundir, prox_config_file_name))
+            if ((not self.configonly) and
+                    self.machine_params['prox_launch_exit']):
+                cmd = 'sudo {}/prox {} -t -o cli -f {}/{}'.format(self.rundir,
+                        autostart, self.rundir, prox_config_file_name)
+                RapidLog.debug("Starting PROX on {}: {}".format(self.name,
+                    cmd))
+                result = self._client.run_cmd(cmd)
+                RapidLog.debug("Finished PROX on {}: {}".format(self.name,
+                    cmd))
 
     def close_prox(self):
-        if (not self.configonly) and self.machine_params['prox_socket'] and self.machine_params['prox_launch_exit']:
+        if (not self.configonly) and self.machine_params[
+                'prox_socket'] and self.machine_params['prox_launch_exit']:
             self.socket.quit_prox()
+            self._client.scp_get('/prox.log', '{}/{}.prox.log'.format(
+                self.resultsdir, self.name))
 
     def connect_prox(self):
         if self.machine_params['prox_socket']:
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py
index d7d8fabc..eba71d96 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py
@@ -53,14 +53,21 @@ class RapidConfigParser(object):
         config = configparser.RawConfigParser()
         config.read(test_params['environment_file'])
         test_params['vim_type'] = config.get('Varia', 'vim')
-        test_params['key'] = config.get('ssh', 'key')
         test_params['user'] = config.get('ssh', 'user')
-        if test_params['user'] in ['rapid']:
-            if test_params['key'] != 'rapid_rsa_key':
-                RapidLog.debug(("Key file {} for user {} overruled by key file:"
-                        " rapid_rsa_key").format(test_params['key'],
-                        test_params['user']))
-                test_params['key'] = 'rapid_rsa_key'
+        if config.has_option('ssh', 'key'):
+            test_params['key'] = config.get('ssh', 'key')
+            if test_params['user'] in ['rapid']:
+                if test_params['key'] != 'rapid_rsa_key':
+                    RapidLog.debug(("Key file {} for user {} overruled by key file:"
+                            " rapid_rsa_key").format(test_params['key'],
+                            test_params['user']))
+                    test_params['key'] = 'rapid_rsa_key'
+        else:
+            test_params['key'] = None
+        if config.has_option('ssh', 'password'):
+            test_params['password'] = config.get('ssh', 'password')
+        else:
+            test_params['password'] = None
         test_params['total_number_of_machines'] = int(config.get('rapid',
             'total_number_of_machines'))
         tests = []
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_sshclient.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_sshclient.py
index e9fe134c..d8aeacc1 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_sshclient.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_sshclient.py
@@ -15,6 +15,7 @@
 ##
 
 import paramiko
+from scp import SCPClient
 import logging
 
 class SSHClient:
@@ -32,9 +33,11 @@ class SSHClient:
     _output = None
     _error = None
 
-    def __init__(self, ip=None, user=None, rsa_private_key=None, timeout=15, logger_name=None):
+    def __init__(self, ip=None, user=None, rsa_private_key=None, timeout=15,
+            logger_name=None, password = None):
         self._ip = ip
         self._user = user
+        self._password = password
         self._rsa_private_key = rsa_private_key
         self._timeout = timeout
 
@@ -43,19 +46,21 @@ class SSHClient:
 
         self._connected = False
 
-    def set_credentials(self, ip, user, rsa_private_key):
+    def set_credentials(self, ip, user, rsa_private_key, password = None):
         self._ip = ip
         self._user = user
+        self._password = password
         self._rsa_private_key = rsa_private_key
 
     def connect(self):
+
         if self._connected:
             if (self._log is not None):
                 self._log.debug("Already connected!")
             return
-
         if ((self._ip is None) or (self._user is None) or
-            (self._rsa_private_key is None)):
+            ((self._rsa_private_key is None) ==
+                (self._password is None))):
             if (self._log is not None):
                 self._log.error("Wrong parameter! IP %s, user %s, RSA private key %s"
                                 % (self._ip, self._user, self._rsa_private_key))
@@ -64,10 +69,14 @@ class SSHClient:
 
         self._ssh = paramiko.SSHClient()
         self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        private_key = paramiko.RSAKey.from_private_key_file(self._rsa_private_key)
+        if (self._rsa_private_key is not None):
+            private_key = paramiko.RSAKey.from_private_key_file(self._rsa_private_key)
+        else:
+            private_key = None
 
         try:
-            self._ssh.connect(hostname = self._ip, username = self._user, pkey = private_key)
+            self._ssh.connect(hostname = self._ip, username = self._user,
+                    password = self._password, pkey = private_key)
         except Exception as e:
             if (self._log is not None):
                 self._log.error("Failed to connect to the host! IP %s, user %s, RSA private key %s\n%s"
@@ -104,6 +113,50 @@ class SSHClient:
 
         return ret
 
+    def scp_put(self, src, dst):
+        self.connect()
+
+        if self._connected is not True:
+            return -1
+
+        try:
+            ret = 0
+            scp = SCPClient(self._ssh.get_transport())
+            scp.put(src, dst)
+            self._output = stdout.read()
+            self._error = stderr.read()
+        except Exception as e:
+            if (self._log is not None):
+                self._log.error("Failed to execute command! IP %s, cmd %s\n%s"
+                                % (self._ip, cmd, e))
+            ret = -1
+
+        self.disconnect()
+
+        return ret
+
+    def scp_get(self, src, dst):
+        self.connect()
+
+        if self._connected is not True:
+            return -1
+
+        try:
+            ret = 0
+            scp = SCPClient(self._ssh.get_transport())
+            scp.get(src, dst)
+            self._output = stdout.read()
+            self._error = stderr.read()
+        except Exception as e:
+            if (self._log is not None):
+                self._log.error("Failed to execute command! IP %s, cmd %s\n%s"
+                                % (self._ip, cmd, e))
+            ret = -1
+
+        self.disconnect()
+
+        return ret
+
     def get_output(self):
         return self._output
 
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py
index 5f78ec01..f3d489d6 100755
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py
@@ -62,9 +62,10 @@ class RapidTestManager(object):
         for machine_params in test_params['machines']:
             if 'gencores' in machine_params.keys():
                 machine = RapidGeneratorMachine(test_params['key'],
-                        test_params['user'], test_params['vim_type'],
-                        test_params['rundir'], test_params['resultsdir'],
-                        machine_params, configonly, test_params['ipv6'])
+                        test_params['user'], test_params['password'],
+                        test_params['vim_type'], test_params['rundir'],
+                        test_params['resultsdir'], machine_params, configonly,
+                        test_params['ipv6'])
                 if machine_params['monitor']:
                     if monitor_gen:
                         RapidLog.exception("Can only monitor 1 generator")
@@ -76,8 +77,9 @@ class RapidTestManager(object):
                     background_machines.append(machine)
             else:
                 machine = RapidMachine(test_params['key'], test_params['user'],
-                        test_params['vim_type'], test_params['rundir'],
-                        test_params['resultsdir'], machine_params, configonly)
+                        test_params['password'], test_params['vim_type'],
+                        test_params['rundir'], test_params['resultsdir'],
+                        machine_params, configonly)
                 if machine_params['monitor']:
                     if monitor_sut:
                         RapidLog.exception("Can only monitor 1 sut")