Complete OVS logging for sfc testcases

- Add archiving capability in ovs_utils
- Refactor common code into a single method
- Add logging for compute nodes
- Rename jumphost to proxy to avoid confusion

Change-Id: I332bcb537d914c25febae53d5d48943e666da4f9
Signed-off-by: George Paraskevopoulos <geopar@intracom-telecom.com>

4 files changed, 191 insertions, 90 deletions

diff --git a/testcases/features/sfc/SSHUtils.py b/testcases/features/sfc/SSHUtils.py
index 55cc5108..9c8c2c72 100644
--- a/testcases/features/sfc/SSHUtils.py
+++ b/testcases/features/sfc/SSHUtils.py
@@ -16,16 +16,16 @@ import os
 logger = rl.Logger('SSHUtils').getLogger()
 
 
-def get_ssh_client(hostname, username, password=None, jumphost=None):
+def get_ssh_client(hostname, username, password=None, proxy=None):
     client = None
     try:
-        if jumphost is None:
+        if proxy is None:
             client = paramiko.SSHClient()
         else:
-            client = JumpHostHopClient()
-            client.configure_jump_host(jumphost['ip'],
-                                       jumphost['username'],
-                                       jumphost['password'])
+            client = ProxyHopClient()
+            client.configure_jump_host(proxy['ip'],
+                                       proxy['username'],
+                                       proxy['password'])
 
         if client is None:
             raise Exception('Could not connect to client')
@@ -62,30 +62,30 @@ def put_file(ssh_conn, src, dest):
         return None
 
 
-class JumpHostHopClient(paramiko.SSHClient):
+class ProxyHopClient(paramiko.SSHClient):
     '''
-    Connect to a remote server using a jumphost hop
+    Connect to a remote server using a proxy hop
     '''
     def __init__(self, *args, **kwargs):
-        self.logger = rl.Logger("JumpHostHopClient").getLogger()
-        self.jumphost_ssh = None
-        self.jumphost_transport = None
-        self.jumphost_channel = None
-        self.jumphost_ip = None
-        self.jumphost_ssh_key = None
+        self.logger = rl.Logger("ProxyHopClient").getLogger()
+        self.proxy_ssh = None
+        self.proxy_transport = None
+        self.proxy_channel = None
+        self.proxy_ip = None
+        self.proxy_ssh_key = None
         self.local_ssh_key = os.path.join(os.getcwd(), 'id_rsa')
-        super(JumpHostHopClient, self).__init__(*args, **kwargs)
+        super(ProxyHopClient, self).__init__(*args, **kwargs)
 
     def configure_jump_host(self, jh_ip, jh_user, jh_pass,
                             jh_ssh_key='/root/.ssh/id_rsa'):
-        self.jumphost_ip = jh_ip
-        self.jumphost_ssh_key = jh_ssh_key
-        self.jumphost_ssh = paramiko.SSHClient()
-        self.jumphost_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        self.jumphost_ssh.connect(jh_ip,
-                                  username=jh_user,
-                                  password=jh_pass)
-        self.jumphost_transport = self.jumphost_ssh.get_transport()
+        self.proxy_ip = jh_ip
+        self.proxy_ssh_key = jh_ssh_key
+        self.proxy_ssh = paramiko.SSHClient()
+        self.proxy_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.proxy_ssh.connect(jh_ip,
+                               username=jh_user,
+                               password=jh_pass)
+        self.proxy_transport = self.proxy_ssh.get_transport()
 
     def connect(self, hostname, port=22, username='root', password=None,
                 pkey=None, key_filename=None, timeout=None, allow_agent=True,
@@ -93,28 +93,28 @@ class JumpHostHopClient(paramiko.SSHClient):
                 gss_kex=False, gss_deleg_creds=True, gss_host=None,
                 banner_timeout=None):
         try:
-            if self.jumphost_ssh is None:
+            if self.proxy_ssh is None:
                 raise Exception('You must configure the jump '
                                 'host before calling connect')
 
-            get_file_res = get_file(self.jumphost_ssh,
-                                    self.jumphost_ssh_key,
+            get_file_res = get_file(self.proxy_ssh,
+                                    self.proxy_ssh_key,
                                     self.local_ssh_key)
             if get_file_res is None:
                 raise Exception('Could\'t fetch SSH key from jump host')
-            jumphost_key = (paramiko.RSAKey
-                            .from_private_key_file(self.local_ssh_key))
+            proxy_key = (paramiko.RSAKey
+                         .from_private_key_file(self.local_ssh_key))
 
-            self.jumphost_channel = self.jumphost_transport.open_channel(
+            self.proxy_channel = self.proxy_transport.open_channel(
                 "direct-tcpip",
                 (hostname, 22),
-                (self.jumphost_ip, 22))
+                (self.proxy_ip, 22))
 
             self.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-            super(JumpHostHopClient, self).connect(hostname,
-                                                   username=username,
-                                                   pkey=jumphost_key,
-                                                   sock=self.jumphost_channel)
+            super(ProxyHopClient, self).connect(hostname,
+                                                username=username,
+                                                pkey=proxy_key,
+                                                sock=self.proxy_channel)
             os.remove(self.local_ssh_key)
         except Exception, e:
             self.logger.error(e)
diff --git a/testcases/features/sfc/ovs_utils.py b/testcases/features/sfc/ovs_utils.py
index 83a36286..29eed384 100644
--- a/testcases/features/sfc/ovs_utils.py
+++ b/testcases/features/sfc/ovs_utils.py
@@ -10,21 +10,29 @@
 import functest.utils.functest_logger as rl
 import os
 import time
+import shutil
 
 logger = rl.Logger('ovs_utils').getLogger()
 
 
 class OVSLogger(object):
-    def __init__(self, basedir):
-        self.ovs_dir = os.path.join(basedir, 'odl-sfc/ovs')
+    def __init__(self, basedir, ft_resdir):
+        self.ovs_dir = basedir
+        self.ft.resdir = ft_resdir
         self.__mkdir_p(self.ovs_dir)
 
     def __mkdir_p(self, dirpath):
         if not os.path.exists(dirpath):
             os.makedirs(dirpath)
 
-    def __ssh_host(self, ssh_conn):
-        return ssh_conn.get_transport().getpeername()[0]
+    def __ssh_host(self, ssh_conn, host_prefix='10.20.0'):
+        try:
+            _, stdout, _ = ssh_conn.exec_command('hostname -I')
+            hosts = stdout.readline().strip().split(' ')
+            found_host = [h for h in hosts if h.startswith(host_prefix)][0]
+            return found_host
+        except Exception, e:
+            logger.error(e)
 
     def __dump_to_file(self, operation, host, text, timestamp=None):
         ts = (timestamp if timestamp is not None
@@ -51,6 +59,13 @@ class OVSLogger(object):
                          .format(cmd, e))
             return None
 
+    def create_artifact_archive(self):
+        shutil.make_archive(self.ovs_dir,
+                            'zip',
+                            root_dir=os.path.dirname(self.ovs_dir),
+                            base_dir=self.ovs_dir)
+        shutil.copy2('{0}.zip'.format(self.ovs_dir), self.ft_resdir)
+
     def ofctl_dump_flows(self, ssh_conn, br='br-int',
                          choose_table=None, timestamp=None):
         try:
@@ -78,3 +93,25 @@ class OVSLogger(object):
         except Exception, e:
             logger.error('[vsctl_show(ssh_client)]: {0}'.format(e))
             return None
+
+    def dump_ovs_logs(self, controller_clients, compute_clients,
+                      related_error=None, timestamp=None):
+        if timestamp is None:
+            timestamp = time.strftime("%Y%m%d-%H%M%S")
+
+        for controller_client in controller_clients:
+            self.ofctl_dump_flows(controller_client,
+                                  timestamp=timestamp)
+            self.vsctl_show(controller_client,
+                            timestamp=timestamp)
+
+        for compute_client in compute_clients:
+            self.ofctl_dump_flows(compute_client,
+                                  timestamp=timestamp)
+            self.vsctl_show(compute_client,
+                            timestamp=timestamp)
+
+        if related_error is not None:
+            dumpdir = os.path.join(self.ovs_dir, timestamp)
+            with open(os.path.join(dumpdir, 'error'), 'w') as f:
+                f.write(related_error)
diff --git a/testcases/features/sfc/sfc.py b/testcases/features/sfc/sfc.py
index 866f92e0..96ea3ab3 100755
--- a/testcases/features/sfc/sfc.py
+++ b/testcases/features/sfc/sfc.py
@@ -2,13 +2,14 @@ import os
 import subprocess
 import sys
 import time
-
 import argparse
 import paramiko
 
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
 import functest.utils.openstack_utils as os_utils
+import SSHUtils as ssh_utils
+import ovs_utils
 
 parser = argparse.ArgumentParser()
 
@@ -21,7 +22,7 @@ args = parser.parse_args()
 """ logging configuration """
 logger = ft_logger.Logger("ODL_SFC").getLogger()
 
-FUNCTEST_RESULTS_DIR = '/home/opnfv/functest/results/'
+FUNCTEST_RESULTS_DIR = '/home/opnfv/functest/results/odl-sfc'
 FUNCTEST_REPO = ft_utils.FUNCTEST_REPO
 
 HOME = os.environ['HOME'] + "/"
@@ -55,6 +56,12 @@ TACKER_CHANGECLASSI = "sfc_change_classi.bash"
 
 ssh_options = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
 
+PROXY = {
+    'ip': '10.20.0.2',
+    'username': 'root',
+    'password': 'r00tme'
+}
+
 
 def check_ssh(ip):
     cmd = "sshpass -p opnfv ssh " + ssh_options + " -q " + ip + " exit"
@@ -83,6 +90,15 @@ def main():
                                stderr=subprocess.PIPE)
     ip_server = process.stdout.readline().rstrip()
 
+    comp_cmd = ("sshpass -p r00tme ssh " + ssh_options + " root@10.20.0.2"
+                " 'fuel node'|grep compute|awk '{print $10}'")
+    logger.info("Executing script to get compute IPs: '%s'" % comp_cmd)
+    process = subprocess.Popen(comp_cmd,
+                               shell=True,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    ip_computes = [ip.strip() for ip in process.stdout.readlines()]
+
     iptable_cmd1 = ("sshpass -p r00tme ssh " + ssh_options + " root@10.20.0.2"
                     " ssh " + ip_server + " iptables -P INPUT ACCEPT ")
     iptable_cmd2 = ("sshpass -p r00tme ssh " + ssh_options + " root@10.20.0.2"
@@ -107,6 +123,20 @@ def main():
     neutron_client = os_utils.get_neutron_client()
     glance_client = os_utils.get_glance_client()
 
+    ovs_logger = ovs_utils.OVSLogger(
+        os.path.join(os.getcwd(), 'ovs-logs'),
+        FUNCTEST_RESULTS_DIR)
+
+    controller_clients = [ssh_utils.get_ssh_client(ip_server,
+                                                   'root',
+                                                   proxy=PROXY)]
+    compute_clients = []
+    for c_ip in ip_computes:
+        c_client = ssh_utils.get_ssh_client(c_ip,
+                                            'root',
+                                            proxy=PROXY)
+        compute_clients.append(c_client)
+
 # Download the image
 
     if not os.path.isfile(IMAGE_PATH):
@@ -383,8 +413,14 @@ def main():
             i = i + 1
             json_results.update({"Test 1: SSH Blocked": "Passed"})
         else:
-            logger.error('\033[91m' + "TEST 1 [FAILED] "
-                         "==> SSH NOT BLOCKED" + '\033[0m')
+            timestamp = time.strftime("%Y%m%d-%H%M%S")
+            error = ('\033[91m' + "TEST 1 [FAILED] "
+                     "==> SSH NOT BLOCKED" + '\033[0m')
+            logger.error(error)
+            ovs_logger.dump_ovs_logs(controller_clients,
+                                     compute_clients,
+                                     related_error=error,
+                                     timestamp=timestamp)
             status = "FAIL"
             json_results.update({"Test 1: SSH Blocked": "Failed"})
             failures += 1
@@ -406,8 +442,14 @@ def main():
             i = i + 1
             json_results.update({"Test 2: HTTP works": "Passed"})
         else:
-            logger.error('\033[91m' + "TEST 2 [FAILED] "
-                         "==> HTTP BLOCKED" + '\033[0m')
+            timestamp = time.strftime("%Y%m%d-%H%M%S")
+            error = ('\033[91m' + "TEST 2 [FAILED] "
+                     "==> HTTP BLOCKED" + '\033[0m')
+            logger.error(error)
+            ovs_logger.dump_ovs_logs(controller_clients,
+                                     compute_clients,
+                                     related_error=error,
+                                     timestamp=timestamp)
             status = "FAIL"
             json_results.update({"Test 2: HTTP works": "Failed"})
             failures += 1
@@ -439,8 +481,14 @@ def main():
             i = i + 1
             json_results.update({"Test 3: HTTP Blocked": "Passed"})
         else:
-            logger.error('\033[91m' + "TEST 3 [FAILED] "
-                         "==> HTTP NOT BLOCKED" + '\033[0m')
+            timestamp = time.strftime("%Y%m%d-%H%M%S")
+            error = ('\033[91m' + "TEST 3 [FAILED] "
+                     "==> HTTP NOT BLOCKED" + '\033[0m')
+            logger.error(error)
+            ovs_logger.dump_ovs_logs(controller_clients,
+                                     compute_clients,
+                                     related_error=error,
+                                     timestamp=timestamp)
             status = "FAIL"
             json_results.update({"Test 3: HTTP Blocked": "Failed"})
             failures += 1
@@ -462,8 +510,14 @@ def main():
             i = i + 1
             json_results.update({"Test 4: SSH works": "Passed"})
         else:
-            logger.error('\033[91m' + "TEST 4 [FAILED] "
-                         "==> SSH BLOCKED" + '\033[0m')
+            timestamp = time.strftime("%Y%m%d-%H%M%S")
+            error = ('\033[91m' + "TEST 4 [FAILED] "
+                     "==> SSH BLOCKED" + '\033[0m')
+            logger.error(error)
+            ovs_logger.dump_ovs_logs(controller_clients,
+                                     compute_clients,
+                                     related_error=error,
+                                     timestamp=timestamp)
             status = "FAIL"
             json_results.update({"Test 4: SSH works": "Failed"})
             failures += 1
@@ -472,6 +526,8 @@ def main():
         time.sleep(6)
         # timeout -= 1
 
+    ovs_logger.create_artifact_archive()
+
     if i == 4:
         for x in range(0, 5):
             logger.info('\033[92m' + "SFC TEST WORKED"
diff --git a/testcases/features/sfc/sfc_colorado1.py b/testcases/features/sfc/sfc_colorado1.py
index 20bfa2e4..3f685e2d 100755
--- a/testcases/features/sfc/sfc_colorado1.py
+++ b/testcases/features/sfc/sfc_colorado1.py
@@ -22,7 +22,7 @@ args = parser.parse_args()
 """ logging configuration """
 logger = ft_logger.Logger("ODL_SFC").getLogger()
 
-FUNCTEST_RESULTS_DIR = '/home/opnfv/functest/results/'
+FUNCTEST_RESULTS_DIR = '/home/opnfv/functest/results/odl-sfc'
 FUNCTEST_REPO = ft_utils.FUNCTEST_REPO
 
 HOME = os.environ['HOME'] + "/"
@@ -56,6 +56,12 @@ TACKER_CHANGECLASSI = "sfc_change_classi.bash"
 
 ssh_options = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
 
+PROXY = {
+    'ip': '10.20.0.2',
+    'username': 'root',
+    'password': 'r00tme'
+}
+
 
 def check_ssh(ip):
     cmd = "sshpass -p opnfv ssh " + ssh_options + " -q " + ip + " exit"
@@ -84,6 +90,15 @@ def main():
                                stderr=subprocess.PIPE)
     ip_server = process.stdout.readline().rstrip()
 
+    comp_cmd = ("sshpass -p r00tme ssh " + ssh_options + " root@10.20.0.2"
+                " 'fuel node'|grep compute|awk '{print $10}'")
+    logger.info("Executing script to get compute IPs: '%s'" % comp_cmd)
+    process = subprocess.Popen(comp_cmd,
+                               shell=True,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    ip_computes = [ip.strip() for ip in process.stdout.readlines()]
+
     iptable_cmd1 = ("sshpass -p r00tme ssh " + ssh_options + " root@10.20.0.2"
                     " ssh " + ip_server + " iptables -P INPUT ACCEPT ")
     iptable_cmd2 = ("sshpass -p r00tme ssh " + ssh_options + " root@10.20.0.2"
@@ -108,15 +123,19 @@ def main():
     neutron_client = os_utils.get_neutron_client()
     glance_client = os_utils.get_glance_client()
 
-    ovs_logger = ovs_utils.OVSLogger(FUNCTEST_RESULTS_DIR)
-    jumphost = {
-        'ip': '10.20.0.2',
-        'username': 'root',
-        'password': 'r00tme'
-    }
-    controller_client = ssh_utils.get_ssh_client(ip_server,
-                                                 'root',
-                                                 jumphost=jumphost)
+    ovs_logger = ovs_utils.OVSLogger(
+        os.path.join(os.getcwd(), 'ovs-logs'),
+        FUNCTEST_RESULTS_DIR)
+
+    controller_clients = [ssh_utils.get_ssh_client(ip_server,
+                                                   'root',
+                                                   proxy=PROXY)]
+    compute_clients = []
+    for c_ip in ip_computes:
+        c_client = ssh_utils.get_ssh_client(c_ip,
+                                            'root',
+                                            proxy=PROXY)
+        compute_clients.append(c_client)
 
 # Download the image
 
@@ -421,14 +440,10 @@ def main():
                 error = ('\033[91m' + "TEST 1 [FAILED] "
                          "==> SSH NOT BLOCKED" + '\033[0m')
                 logger.error(error)
-                ovs_logger.ofctl_dump_flows(controller_client,
-                                            timestamp=timestamp)
-                ovs_logger.vsctl_show(controller_client,
-                                      timestamp=timestamp)
-
-                dumpdir = os.path.join(ovs_logger.ovs_dir, timestamp)
-                with open(os.path.join(dumpdir, 'error'), 'w') as f:
-                    f.write(error)
+                ovs_logger.dump_ovs_logs(controller_clients,
+                                         compute_clients,
+                                         related_error=error,
+                                         timestamp=timestamp)
                 status = "FAIL"
                 json_results.update({"Test 1: SSH Blocked": "Failed"})
                 failures += 1
@@ -450,17 +465,14 @@ def main():
                 i = i + 1
                 json_results.update({"Test 2: HTTP works": "Passed"})
             else:
+                timestamp = time.strftime("%Y%m%d-%H%M%S")
                 error = ('\033[91m' + "TEST 2 [FAILED] "
                          "==> HTTP BLOCKED" + '\033[0m')
                 logger.error(error)
-                ovs_logger.ofctl_dump_flows(controller_client,
-                                            timestamp=timestamp)
-                ovs_logger.vsctl_show(controller_client,
-                                      timestamp=timestamp)
-
-                dumpdir = os.path.join(ovs_logger.ovs_dir, timestamp)
-                with open(os.path.join(dumpdir, 'error'), 'w') as f:
-                    f.write(error)
+                ovs_logger.dump_ovs_logs(controller_clients,
+                                         compute_clients,
+                                         related_error=error,
+                                         timestamp=timestamp)
                 status = "FAIL"
                 json_results.update({"Test 2: HTTP works": "Failed"})
                 failures += 1
@@ -492,17 +504,14 @@ def main():
                 i = i + 1
                 json_results.update({"Test 3: HTTP Blocked": "Passed"})
             else:
+                timestamp = time.strftime("%Y%m%d-%H%M%S")
                 error = ('\033[91m' + "TEST 3 [FAILED] "
                          "==> HTTP NOT BLOCKED" + '\033[0m')
                 logger.error(error)
-                ovs_logger.ofctl_dump_flows(controller_client,
-                                            timestamp=timestamp)
-                ovs_logger.vsctl_show(controller_client,
-                                      timestamp=timestamp)
-
-                dumpdir = os.path.join(ovs_logger.ovs_dir, timestamp)
-                with open(os.path.join(dumpdir, 'error'), 'w') as f:
-                    f.write(error)
+                ovs_logger.dump_ovs_logs(controller_clients,
+                                         compute_clients,
+                                         related_error=error,
+                                         timestamp=timestamp)
                 status = "FAIL"
                 json_results.update({"Test 3: HTTP Blocked": "Failed"})
                 failures += 1
@@ -524,17 +533,14 @@ def main():
                 i = i + 1
                 json_results.update({"Test 4: SSH works": "Passed"})
             else:
+                timestamp = time.strftime("%Y%m%d-%H%M%S")
                 error = ('\033[91m' + "TEST 4 [FAILED] "
                          "==> SSH BLOCKED" + '\033[0m')
                 logger.error(error)
-                ovs_logger.ofctl_dump_flows(controller_client,
-                                            timestamp=timestamp)
-                ovs_logger.vsctl_show(controller_client,
-                                      timestamp=timestamp)
-
-                dumpdir = os.path.join(ovs_logger.ovs_dir, timestamp)
-                with open(os.path.join(dumpdir, 'error'), 'w') as f:
-                    f.write(error)
+                ovs_logger.dump_ovs_logs(controller_clients,
+                                         compute_clients,
+                                         related_error=error,
+                                         timestamp=timestamp)
                 status = "FAIL"
                 json_results.update({"Test 4: SSH works": "Failed"})
                 failures += 1
@@ -543,6 +549,8 @@ def main():
             time.sleep(6)
             # timeout -= 1
 
+        ovs_logger.create_artifact_archive()
+
         iterator += 1
         if i == 4:
             for x in range(0, 5):