Fix SSH client connection reset

SSH connection might be reseted with long running session.
This may happen if run "testcase=all"

Change-Id: I232ae906628411dfbe0bbdbdc8d4fb43167760fd
Signed-off-by: Tomi Juvonen <tomi.juvonen@nokia.com>
(cherry picked from commit 2ca5924081ce4784f599437707bd32807aa155ce)

1 files changed, 30 insertions, 11 deletions

diff --git a/doctor_tests/installer/apex.py b/doctor_tests/installer/apex.py
index 9b0010e4..2aa81ff9 100644
--- a/doctor_tests/installer/apex.py
+++ b/doctor_tests/installer/apex.py
@@ -192,20 +192,39 @@ class ApexInstaller(BaseInstaller):
                 restart_cmd += ' openstack-congress-server.service'
             restore_scripts.append(self.cg_restore_script)
 
-        for client in self.controller_clients:
-            self._run_apply_patches(client,
-                                    restart_cmd,
-                                    restore_scripts,
-                                    python=self.python)
-
+        for client, node_ip in zip(self.controller_clients, self.controllers):
+            retry = 0
+            while retry < 2:
+                try:
+                    self._run_apply_patches(client,
+                                            restart_cmd,
+                                            restore_scripts,
+                                            python=self.python)
+                except Exception:
+                    if retry > 0:
+                        raise Exception("SSHClient to %s feiled" % node_ip)
+                    client = SSHClient(node_ip, self.node_user_name,
+                                       key_filename=self.key_file)
+                    retry += 1
+                break
         if self.conf.test_case != 'fault_management':
             if self.use_containers:
                 restart_cmd = self._set_docker_restart_cmd("nova-compute")
             else:
                 restart_cmd = 'sudo systemctl restart' \
                               ' openstack-nova-compute.service'
-            for client in self.compute_clients:
-                self._run_apply_patches(client,
-                                        restart_cmd,
-                                        [self.nc_restore_compute_script],
-                                        python=self.python)
+            for client, node_ip in zip(self.compute_clients, self.computes):
+                retry = 0
+                while retry < 2:
+                    try:
+                        self._run_apply_patches(
+                            client, restart_cmd,
+                            [self.nc_restore_compute_script],
+                            python=self.python)
+                    except Exception:
+                        if retry > 0:
+                            raise Exception("SSHClient to %s feiled" % node_ip)
+                        client = SSHClient(node_ip, self.node_user_name,
+                                           key_filename=self.key_file)
+                        retry += 1
+                    break