Support Fenix and sample implementation accordingly

Fenix has now same capability as our sample implementation.
We can now support Fenix if manually installed on controllers.
Sample implementation should be closer to Fenix as Fenix is the
place to define the generic interfaces at the end.

JIRA: DOCTOR-131

Change-Id: Ied58b8f469dbcc4bb5caa787e62c1831a211ecd6
Signed-off-by: Tomi Juvonen <tomi.juvonen@nokia.com>

3 files changed, 46 insertions, 22 deletions

diff --git a/doctor_tests/installer/apex.py b/doctor_tests/installer/apex.py
index 2aa81ff9..3c97378c 100644
--- a/doctor_tests/installer/apex.py
+++ b/doctor_tests/installer/apex.py
@@ -114,6 +114,22 @@ class ApexInstaller(BaseInstaller):
     def set_apply_patches(self):
         self.log.info('Set apply patches start......')
 
+        if self.conf.test_case != 'fault_management':
+            if self.use_containers:
+                restart_cmd = self._set_docker_restart_cmd("nova-compute")
+            else:
+                restart_cmd = 'sudo systemctl restart' \
+                              ' openstack-nova-compute.service'
+            for node_ip in self.computes:
+                client = SSHClient(node_ip, self.node_user_name,
+                                   key_filename=self.key_file)
+                self.compute_clients.append(client)
+                self._run_apply_patches(client,
+                                        restart_cmd,
+                                        [self.nc_set_compute_script],
+                                        python=self.python)
+            time.sleep(10)
+
         set_scripts = [self.cm_set_script]
 
         if self.use_containers:
@@ -147,24 +163,6 @@ class ApexInstaller(BaseInstaller):
                                     set_scripts,
                                     python=self.python)
 
-        if self.conf.test_case != 'fault_management':
-            if self.use_containers:
-                restart_cmd = self._set_docker_restart_cmd("nova-compute")
-            else:
-                restart_cmd = 'sudo systemctl restart' \
-                              ' openstack-nova-compute.service'
-            for node_ip in self.computes:
-                client = SSHClient(node_ip, self.node_user_name,
-                                   key_filename=self.key_file)
-                self.compute_clients.append(client)
-                self._run_apply_patches(client,
-                                        restart_cmd,
-                                        [self.nc_set_compute_script],
-                                        python=self.python)
-
-        if self.conf.test_case != 'fault_management':
-            time.sleep(10)
-
     def restore_apply_patches(self):
         self.log.info('restore apply patches start......')
 
diff --git a/doctor_tests/installer/base.py b/doctor_tests/installer/base.py
index 30435931..124b1910 100644
--- a/doctor_tests/installer/base.py
+++ b/doctor_tests/installer/base.py
@@ -14,8 +14,9 @@ import pwd
 import six
 import stat
 import subprocess
+import time
 
-from doctor_tests.common.utils import get_doctor_test_root_dir
+from doctor_tests.common import utils
 from doctor_tests.identity_auth import get_session
 from doctor_tests.os_clients import nova_client
 
@@ -75,7 +76,7 @@ class BaseInstaller(object):
                 cmd = ("ssh -o UserKnownHostsFile=/dev/null"
                        " -o StrictHostKeyChecking=no"
                        " -i %s %s@%s -R %s:localhost:%s"
-                       " sleep %s > ssh_tunnel.%s"
+                       " sleep %s > ssh_tunnel.%s.%s"
                        " 2>&1 < /dev/null "
                        % (self.key_file,
                           self.node_user_name,
@@ -83,9 +84,28 @@ class BaseInstaller(object):
                           port,
                           port,
                           tunnel_uptime,
-                          node_ip))
+                          node_ip,
+                          port))
                 server = subprocess.Popen('exec ' + cmd, shell=True)
                 self.servers.append(server)
+        if self.conf.admin_tool.type == 'fenix':
+            port = self.conf.admin_tool.port
+            self.log.info('tunnel for port %s' % port)
+            cmd = ("ssh -o UserKnownHostsFile=/dev/null"
+                   " -o StrictHostKeyChecking=no"
+                   " -i %s %s@%s -L %s:localhost:%s"
+                   " sleep %s > ssh_tunnel.%s.%s"
+                   " 2>&1 < /dev/null "
+                   % (self.key_file,
+                      self.node_user_name,
+                      node_ip,
+                      port,
+                      port,
+                      tunnel_uptime,
+                      node_ip,
+                      port))
+            server = subprocess.Popen('exec ' + cmd, shell=True)
+            self.servers.append(server)
 
     def _get_ssh_key(self, client, key_path):
         self.log.info('Get SSH keys from %s installer......'
@@ -96,7 +116,8 @@ class BaseInstaller(object):
                           % self.conf.installer.type)
             return self.key_file
 
-        ssh_key = '{0}/{1}'.format(get_doctor_test_root_dir(), 'instack_key')
+        ssh_key = '{0}/{1}'.format(utils.get_doctor_test_root_dir(),
+                                   'instack_key')
         client.scp(key_path, ssh_key, method='get')
         user = getpass.getuser()
         uid = pwd.getpwnam(user).pw_uid
@@ -131,6 +152,7 @@ class BaseInstaller(object):
             ret = False
         return ret
 
+    @utils.run_async
     def _run_apply_patches(self, client, restart_cmd, script_names,
                            python='python3'):
         installer_dir = os.path.dirname(os.path.realpath(__file__))
@@ -146,4 +168,7 @@ class BaseInstaller(object):
                     raise Exception('Do the command in remote'
                                     ' node failed, ret=%s, cmd=%s, output=%s'
                                     % (ret, cmd, output))
+            if 'nova-scheduler' in restart_cmd:
+                # Make sure scheduler has proper cpu_allocation_ratio
+                time.sleep(5)
             client.ssh(restart_cmd)
diff --git a/doctor_tests/installer/common/set_config.py b/doctor_tests/installer/common/set_config.py
index 3dc6cd9a..e66d4c2c 100644
--- a/doctor_tests/installer/common/set_config.py
+++ b/doctor_tests/installer/common/set_config.py
@@ -125,6 +125,7 @@ def set_event_definitions():
                 'reply_url': {'fields': 'payload.reply_url'},
                 'actions_at': {'fields': 'payload.actions_at',
                                'type': 'datetime'},
+                'reply_at': {'fields': 'payload.reply_at', 'type': 'datetime'},
                 'state': {'fields': 'payload.state'},
                 'session_id': {'fields': 'payload.session_id'},
                 'project_id': {'fields': 'payload.project_id'},