Support Apex with services in containers

Upstream apex now works in containers, so if used, it needs to be supported

JIRA: DOCTOR-130

Change-Id: I3d73a1699e4fee53b001f043f55d0eeefa7bfb7b
Signed-off-by: Tomi Juvonen <tomi.juvonen@nokia.com>

14 files changed, 316 insertions, 117 deletions

diff --git a/doctor_tests/common/utils.py b/doctor_tests/common/utils.py
index 1a84c824..1a8840dd 100644
--- a/doctor_tests/common/utils.py
+++ b/doctor_tests/common/utils.py
@@ -67,7 +67,7 @@ class SSHClient(object):
     def __del__(self):
         self.client.close()
 
-    def ssh(self, command):
+    def ssh(self, command, raise_enabled=True):
         if self.log:
             self.log.info("Executing: %s" % command)
         stdin, stdout, stderr = self.client.exec_command(command)
@@ -75,7 +75,7 @@ class SSHClient(object):
         output = list()
         for line in stdout.read().splitlines():
             output.append(line.decode('utf-8'))
-        if ret:
+        if ret and raise_enabled:
             if self.log:
                 self.log.info("*** FAILED to run command %s (%s)"
                               % (command, ret))
diff --git a/doctor_tests/installer/apex.py b/doctor_tests/installer/apex.py
index 694adb88..9b0010e4 100644
--- a/doctor_tests/installer/apex.py
+++ b/doctor_tests/installer/apex.py
@@ -16,17 +16,21 @@ from doctor_tests.installer.base import BaseInstaller
 
 class ApexInstaller(BaseInstaller):
     node_user_name = 'heat-admin'
+    installer_username = 'stack'
     cm_set_script = 'set_config.py'
     nc_set_compute_script = 'set_compute_config.py'
     cg_set_script = 'set_congress.py'
     cm_restore_script = 'restore_config.py'
     nc_restore_compute_script = 'restore_compute_config.py'
     cg_restore_script = 'restore_congress.py'
+    ac_restart_script = 'restart_aodh.py'
+    ac_restore_script = 'restore_aodh.py'
+    python = 'python'
 
     def __init__(self, conf, log):
         super(ApexInstaller, self).__init__(conf, log)
         self.client = SSHClient(self.conf.installer.ip,
-                                self.conf.installer.username,
+                                self.installer_username,
                                 key_filename=self.conf.installer.key_file,
                                 look_for_keys=True)
         self.key_file = None
@@ -38,7 +42,7 @@ class ApexInstaller(BaseInstaller):
     def setup(self):
         self.log.info('Setup Apex installer start......')
         self.key_file = self.get_ssh_key_from_installer()
-        self._get_and_set_ips()
+        self._get_overcloud_conf()
         self.create_flavor()
         self.set_apply_patches()
         self.setup_stunnel()
@@ -52,8 +56,8 @@ class ApexInstaller(BaseInstaller):
         key_path = '/home/stack/.ssh/id_rsa'
         return self._get_ssh_key(self.client, key_path)
 
-    def _get_and_set_ips(self):
-        self.log.info('Get controller and compute ips from Apex installer'
+    def _get_overcloud_conf(self):
+        self.log.info('Get overcloud config details from Apex installer'
                       '......')
 
         command = "source stackrc; nova list | grep ' overcloud-'"
@@ -64,8 +68,11 @@ class ApexInstaller(BaseInstaller):
                 self.controllers.append(ip)
             elif 'overcloud-novacompute-' in line:
                 self.computes.append(ip)
+        command = "grep docker /home/stack/deploy_command"
+        self.use_containers = self._check_cmd_remote(self.client, command)
         self.log.info('controller_ips:%s' % self.controllers)
         self.log.info('compute_ips:%s' % self.computes)
+        self.log.info('use_containers:%s' % self.use_containers)
 
     def get_host_ip_from_hostname(self, hostname):
         self.log.info('Get host ip by hostname=%s from Apex installer......'
@@ -79,8 +86,12 @@ class ApexInstaller(BaseInstaller):
     def get_transport_url(self):
         client = SSHClient(self.controllers[0], self.node_user_name,
                            key_filename=self.key_file)
+        if self.use_containers:
+            ncbase = "/var/lib/config-data/puppet-generated/nova"
+        else:
+            ncbase = ""
+        command = 'sudo grep "^transport_url" %s/etc/nova/nova.conf' % ncbase
 
-        command = 'sudo grep "^transport_url" /etc/nova/nova.conf'
         ret, url = client.ssh(command)
         if ret:
             raise Exception('Exec command to get host ip from controller(%s)'
@@ -92,19 +103,39 @@ class ApexInstaller(BaseInstaller):
         self.log.debug('get_transport_url %s' % ret)
         return ret
 
+    def _set_docker_restart_cmd(self, service):
+        # There can be multiple instances running so need to restart all
+        cmd = "for container in `sudo docker ps | grep "
+        cmd += service
+        cmd += " | awk '{print $1}'`; do sudo docker restart $container; \
+               done;"
+        return cmd
+
     def set_apply_patches(self):
         self.log.info('Set apply patches start......')
 
-        restart_cmd = 'sudo systemctl restart' \
-                      ' openstack-ceilometer-notification.service'
-
         set_scripts = [self.cm_set_script]
 
+        if self.use_containers:
+            restart_cmd = (self._set_docker_restart_cmd(
+                           "ceilometer-notification"))
+            set_scripts.append(self.ac_restart_script)
+        else:
+            restart_cmd = 'sudo systemctl restart' \
+                          ' openstack-ceilometer-notification.service'
+
         if self.conf.test_case != 'fault_management':
-            restart_cmd += ' openstack-nova-scheduler.service'
+            if self.use_containers:
+                restart_cmd += self._set_docker_restart_cmd("nova-scheduler")
+            else:
+                restart_cmd += ' openstack-nova-scheduler.service'
+            set_scripts.append(self.nc_set_compute_script)
 
         if self.conf.inspector.type == Inspector.CONGRESS:
-            restart_cmd += ' openstack-congress-server.service'
+            if self.use_containers:
+                restart_cmd += self._set_docker_restart_cmd("congress-server")
+            else:
+                restart_cmd += ' openstack-congress-server.service'
             set_scripts.append(self.cg_set_script)
 
         for node_ip in self.controllers:
@@ -113,18 +144,23 @@ class ApexInstaller(BaseInstaller):
             self.controller_clients.append(client)
             self._run_apply_patches(client,
                                     restart_cmd,
-                                    set_scripts)
+                                    set_scripts,
+                                    python=self.python)
 
         if self.conf.test_case != 'fault_management':
-            restart_cmd = 'sudo systemctl restart' \
-                          ' openstack-nova-compute.service'
+            if self.use_containers:
+                restart_cmd = self._set_docker_restart_cmd("nova-compute")
+            else:
+                restart_cmd = 'sudo systemctl restart' \
+                              ' openstack-nova-compute.service'
             for node_ip in self.computes:
                 client = SSHClient(node_ip, self.node_user_name,
                                    key_filename=self.key_file)
                 self.compute_clients.append(client)
                 self._run_apply_patches(client,
                                         restart_cmd,
-                                        [self.nc_set_compute_script])
+                                        [self.nc_set_compute_script],
+                                        python=self.python)
 
         if self.conf.test_case != 'fault_management':
             time.sleep(10)
@@ -132,27 +168,44 @@ class ApexInstaller(BaseInstaller):
     def restore_apply_patches(self):
         self.log.info('restore apply patches start......')
 
-        restart_cmd = 'sudo systemctl restart' \
-                      ' openstack-ceilometer-notification.service'
-
         restore_scripts = [self.cm_restore_script]
 
+        if self.use_containers:
+            restart_cmd = (self._set_docker_restart_cmd(
+                           "ceilometer-notification"))
+            restore_scripts.append(self.ac_restore_script)
+        else:
+            restart_cmd = 'sudo systemctl restart' \
+                          ' openstack-ceilometer-notification.service'
+
         if self.conf.test_case != 'fault_management':
-            restart_cmd += ' openstack-nova-scheduler.service'
+            if self.use_containers:
+                restart_cmd += self._set_docker_restart_cmd("nova-scheduler")
+            else:
+                restart_cmd += ' openstack-nova-scheduler.service'
+            restore_scripts.append(self.nc_restore_compute_script)
 
         if self.conf.inspector.type == Inspector.CONGRESS:
-            restart_cmd += ' openstack-congress-server.service'
+            if self.use_containers:
+                restart_cmd += self._set_docker_restart_cmd("congress-server")
+            else:
+                restart_cmd += ' openstack-congress-server.service'
             restore_scripts.append(self.cg_restore_script)
 
         for client in self.controller_clients:
             self._run_apply_patches(client,
                                     restart_cmd,
-                                    restore_scripts)
+                                    restore_scripts,
+                                    python=self.python)
 
         if self.conf.test_case != 'fault_management':
-            restart_cmd = 'sudo systemctl restart' \
-                          ' openstack-nova-compute.service'
+            if self.use_containers:
+                restart_cmd = self._set_docker_restart_cmd("nova-compute")
+            else:
+                restart_cmd = 'sudo systemctl restart' \
+                              ' openstack-nova-compute.service'
             for client in self.compute_clients:
                 self._run_apply_patches(client,
                                         restart_cmd,
-                                        [self.nc_restore_compute_script])
+                                        [self.nc_restore_compute_script],
+                                        python=self.python)
diff --git a/doctor_tests/installer/base.py b/doctor_tests/installer/base.py
index 953b36d9..30435931 100644
--- a/doctor_tests/installer/base.py
+++ b/doctor_tests/installer/base.py
@@ -26,6 +26,7 @@ class BaseInstaller(object):
         self.conf = conf
         self.log = log
         self.servers = list()
+        self.use_containers = False
 
     @abc.abstractproperty
     def node_user_name(self):
@@ -118,7 +119,20 @@ class BaseInstaller(object):
                       % (output, command, self.conf.installer.type))
         return output
 
-    def _run_apply_patches(self, client, restart_cmd, script_names):
+    def _check_cmd_remote(self, client, command):
+        self.log.info('Check command=%s return in %s installer......'
+                      % (command, self.conf.installer.type))
+
+        ret, output = client.ssh(command, raise_enabled=False)
+        self.log.info('return %s' % ret)
+        if ret == 0:
+            ret = True
+        else:
+            ret = False
+        return ret
+
+    def _run_apply_patches(self, client, restart_cmd, script_names,
+                           python='python3'):
         installer_dir = os.path.dirname(os.path.realpath(__file__))
 
         if isinstance(script_names, list):
@@ -126,10 +140,10 @@ class BaseInstaller(object):
                 script_abs_path = '{0}/{1}/{2}'.format(installer_dir,
                                                        'common', script_name)
                 client.scp(script_abs_path, script_name)
-                cmd = 'sudo python3 %s' % script_name
+                cmd = 'sudo %s %s' % (python, script_name)
                 ret, output = client.ssh(cmd)
                 if ret:
-                    raise Exception('Do the command in controller'
+                    raise Exception('Do the command in remote'
                                     ' node failed, ret=%s, cmd=%s, output=%s'
                                     % (ret, cmd, output))
             client.ssh(restart_cmd)
diff --git a/doctor_tests/installer/common/restart_aodh.py b/doctor_tests/installer/common/restart_aodh.py
new file mode 100644
index 00000000..4473bdca
--- /dev/null
+++ b/doctor_tests/installer/common/restart_aodh.py
@@ -0,0 +1,42 @@
+##############################################################################
+# Copyright (c) 2018 Nokia Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import socket
+import subprocess
+
+
+def restart_aodh_event_alarm():
+    # Restart aodh-evaluator docker with localhost as controller host ip
+    # This makes our alarm sending look the same as without container
+
+    orig_docker_id = subprocess.check_output("docker ps | grep aodh-evaluator "
+                                             "| awk '{print $1}'", shell=True)
+    get_docker_startup = (
+        'docker inspect --format=\'{{range .Config.Env}} -e "{{.}}" {{end}} '
+        '{{range .Mounts}} -v {{.Source}}:{{.Destination}}{{if .Mode}}:'
+        '{{.Mode}}{{end}}{{end}} -ti {{.Config.Image}}\''
+    )
+    docker_start = subprocess.check_output("%s %s" % (get_docker_startup,
+                                           orig_docker_id), shell=True)
+    with open("orig_docker_id", 'w') as oid:
+        oid.write(orig_docker_id)
+    oid.close()
+    subprocess.check_output("docker stop %s" % orig_docker_id, shell=True)
+    ip = socket.gethostbyname(socket.gethostname())
+
+    ae_start = '-d --add-host="localhost:%s" %s' % (ip, docker_start)
+    subprocess.check_output("docker run %s" % ae_start, shell=True)
+    new_docker_id = subprocess.check_output("docker ps | grep aodh-evaluator "
+                                            " | awk '{print $1}'", shell=True)
+    if orig_docker_id == new_docker_id:
+        raise Exception("Docker ids matching!")
+    with open("new_docker_id", 'w') as nid:
+        nid.write(new_docker_id)
+    nid.close()
+
+restart_aodh_event_alarm()
diff --git a/doctor_tests/installer/common/restore_aodh.py b/doctor_tests/installer/common/restore_aodh.py
new file mode 100644
index 00000000..b55eae8d
--- /dev/null
+++ b/doctor_tests/installer/common/restore_aodh.py
@@ -0,0 +1,32 @@
+##############################################################################
+# Copyright (c) 2018 Nokia Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import subprocess
+
+
+def restore_aodh_event_alarm():
+    # Remove modified docker and restore original
+    orig = "orig_docker_id"
+    new = "new_docker_id"
+    if os.path.isfile(orig):
+        with open("orig_docker_id", 'r') as oid:
+            orig_docker_id = oid.read()
+        oid.close()
+        if os.path.isfile(new):
+            with open("new_docker_id", 'r') as nid:
+                new_docker_id = nid.read()
+            nid.close()
+            subprocess.check_output("docker stop %s" % new_docker_id,
+                                    shell=True)
+            subprocess.check_output("docker rm %s" % new_docker_id, shell=True)
+            os.remove(new)
+        subprocess.check_output("docker start %s" % orig_docker_id, shell=True)
+        os.remove(orig)
+
+restore_aodh_event_alarm()
diff --git a/doctor_tests/installer/common/restore_compute_config.py b/doctor_tests/installer/common/restore_compute_config.py
index 0971d12b..0e9939fd 100644
--- a/doctor_tests/installer/common/restore_compute_config.py
+++ b/doctor_tests/installer/common/restore_compute_config.py
@@ -11,8 +11,11 @@ import shutil
 
 
 def restore_cpu_allocation_ratio():
-    nova_file = '/etc/nova/nova.conf'
-    nova_file_bak = '/etc/nova/nova.bak'
+    nova_base = "/var/lib/config-data/puppet-generated/nova"
+    if not os.path.isdir(nova_base):
+        nova_base = ""
+    nova_file = nova_base + '/etc/nova/nova.conf'
+    nova_file_bak = nova_base + '/etc/nova/nova.bak'
 
     if not os.path.isfile(nova_file_bak):
         print('Bak_file:%s does not exist.' % nova_file_bak)
diff --git a/doctor_tests/installer/common/restore_config.py b/doctor_tests/installer/common/restore_config.py
index c1f919c1..5cb83b27 100644
--- a/doctor_tests/installer/common/restore_config.py
+++ b/doctor_tests/installer/common/restore_config.py
@@ -9,11 +9,15 @@
 import os
 import shutil
 
-ep_file = '/etc/ceilometer/event_pipeline.yaml'
-ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+
+cbase = "/var/lib/config-data/puppet-generated/ceilometer"
+if not os.path.isdir(cbase):
+    cbase = ""
 
 
 def restore_ep_config():
+    ep_file = cbase + '/etc/ceilometer/event_pipeline.yaml'
+    ep_file_bak = cbase + '/etc/ceilometer/event_pipeline.yaml.bak'
 
     if not os.path.isfile(ep_file_bak):
         print('Bak_file:%s does not exist.' % ep_file_bak)
@@ -25,31 +29,20 @@ def restore_ep_config():
 
 
 def restore_ed_config():
-
-    ed_file = '/etc/ceilometer/event_definitions.yaml'
-    ed_file_bak = '/etc/ceilometer/event_definitions.bak'
+    ed_file = cbase + '/etc/ceilometer/event_definitions.yaml'
+    ed_file_bak = cbase + '/etc/ceilometer/event_definitions.bak'
 
     if not os.path.isfile(ed_file_bak):
         print("Bak_file doesn't exist: %s." % ed_file_bak)
     else:
         print('restore: %s' % ed_file)
-        shutil.copyfile(ed_file_bak, ed_file)
+        if os.stat(ed_file_bak).st_size == 0:
+            print('Bak_file empty, so removing also: %s' % ed_file)
+            os.remove(ed_file)
+        else:
+            shutil.copyfile(ed_file_bak, ed_file)
         os.remove(ed_file_bak)
     return
 
-
-def restore_cpu_allocation_ratio():
-    nova_file = '/etc/nova/nova.conf'
-    nova_file_bak = '/etc/nova/nova.bak'
-
-    if not os.path.isfile(nova_file_bak):
-        print('Bak_file:%s does not exist.' % nova_file_bak)
-    else:
-        print('restore: %s' % nova_file)
-        shutil.copyfile(nova_file_bak, nova_file)
-        os.remove(nova_file_bak)
-    return
-
 restore_ep_config()
 restore_ed_config()
-restore_cpu_allocation_ratio()
diff --git a/doctor_tests/installer/common/restore_congress.py b/doctor_tests/installer/common/restore_congress.py
index b5efb1ef..576f1b16 100644
--- a/doctor_tests/installer/common/restore_congress.py
+++ b/doctor_tests/installer/common/restore_congress.py
@@ -11,8 +11,11 @@ import shutil
 
 
 def restore_drivers_config():
-    co_conf = "/etc/congress/congress.conf"
-    co_conf_bak = "/etc/congress/congress.conf.bak"
+    co_base = "/var/lib/config-data/puppet-generated/congress"
+    if not os.path.isdir(co_base):
+        co_base = ""
+    co_conf = co_base + "/etc/congress/congress.conf"
+    co_conf_bak = co_base + "/etc/congress/congress.conf.bak"
 
     if not os.path.isfile(co_conf_bak):
         print('Bak_file:%s does not exist.' % co_conf_bak)
diff --git a/doctor_tests/installer/common/set_compute_config.py b/doctor_tests/installer/common/set_compute_config.py
index 07db1e16..86266085 100644
--- a/doctor_tests/installer/common/set_compute_config.py
+++ b/doctor_tests/installer/common/set_compute_config.py
@@ -10,9 +10,26 @@ import os
 import shutil
 
 
+def make_initial_config(service, dest):
+    for mk in ["", "/etc", "/%s" % service]:
+        dest += mk
+        os.mkdir(dest)
+    src = "/etc/%s/%s.conf" % (service, service)
+    dest += "/%s.conf" % service
+    shutil.copyfile(src, dest)
+
+
 def set_cpu_allocation_ratio():
-    nova_file = '/etc/nova/nova.conf'
-    nova_file_bak = '/etc/nova/nova.bak'
+    docker_conf_base_dir = "/var/lib/config-data/puppet-generated"
+    if not os.path.isdir(docker_conf_base_dir):
+        nova_base = ""
+    else:
+        nova_base = "%s/nova" % docker_conf_base_dir
+        if not os.path.isdir(nova_base):
+            # nova.conf to be used might not exist
+            make_initial_config("nova", nova_base)
+    nova_file = nova_base + '/etc/nova/nova.conf'
+    nova_file_bak = nova_base + '/etc/nova/nova.bak'
 
     if not os.path.isfile(nova_file):
         raise Exception("File doesn't exist: %s." % nova_file)
diff --git a/doctor_tests/installer/common/set_config.py b/doctor_tests/installer/common/set_config.py
index 42465247..3dc6cd9a 100644
--- a/doctor_tests/installer/common/set_config.py
+++ b/doctor_tests/installer/common/set_config.py
@@ -10,12 +10,16 @@ import os
 import shutil
 import yaml
 
-ep_file = '/etc/ceilometer/event_pipeline.yaml'
-ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
-event_notifier_topic = 'notifier://?topic=alarm.all'
+
+cbase = "/var/lib/config-data/puppet-generated/ceilometer"
+if not os.path.isdir(cbase):
+    cbase = ""
 
 
 def set_notifier_topic():
+    ep_file = cbase + '/etc/ceilometer/event_pipeline.yaml'
+    ep_file_bak = cbase + '/etc/ceilometer/event_pipeline.yaml.bak'
+    event_notifier_topic = 'notifier://?topic=alarm.all'
     config_modified = False
 
     if not os.path.isfile(ep_file):
@@ -42,24 +46,77 @@ def set_notifier_topic():
             file.write(yaml.safe_dump(config))
 
 
-def set_maintenance_event_definitions():
-    ed_file = '/etc/ceilometer/event_definitions.yaml'
-    ed_file_bak = '/etc/ceilometer/event_definitions.bak'
+def set_event_definitions():
+    ed_file = cbase + '/etc/ceilometer/event_definitions.yaml'
+    ed_file_bak = cbase + '/etc/ceilometer/event_definitions.bak'
+    orig_ed_file_exist = True
+    modify_config = False
 
     if not os.path.isfile(ed_file):
-        raise Exception("File doesn't exist: %s." % ed_file)
-
-    with open(ed_file, 'r') as file:
-        config = yaml.safe_load(file)
+        # Deployment did not modify file, so it did not exist
+        src_file = '/etc/ceilometer/event_definitions.yaml'
+        if not os.path.isfile(src_file):
+            config = []
+            orig_ed_file_exist = False
+        else:
+            shutil.copyfile('/etc/ceilometer/event_definitions.yaml', ed_file)
+    if orig_ed_file_exist:
+        with open(ed_file, 'r') as file:
+            config = yaml.safe_load(file)
 
     et_list = [et['event_type'] for et in config]
 
+    if 'compute.instance.update' in et_list:
+        print('NOTE: compute.instance.update allready configured')
+    else:
+        print('NOTE: add compute.instance.update to event_definitions.yaml')
+        modify_config = True
+        instance_update = {
+            'event_type': 'compute.instance.update',
+            'traits': {
+                'deleted_at': {'fields': 'payload.deleted_at',
+                               'type': 'datetime'},
+                'disk_gb': {'fields': 'payload.disk_gb',
+                            'type': 'int'},
+                'display_name': {'fields': 'payload.display_name'},
+                'ephemeral_gb': {'fields': 'payload.ephemeral_gb',
+                                 'type': 'int'},
+                'host': {'fields': 'publisher_id.`split(., 1, 1)`'},
+                'instance_id': {'fields': 'payload.instance_id'},
+                'instance_type': {'fields': 'payload.instance_type'},
+                'instance_type_id': {'fields': 'payload.instance_type_id',
+                                     'type': 'int'},
+                'launched_at': {'fields': 'payload.launched_at',
+                                'type': 'datetime'},
+                'memory_mb': {'fields': 'payload.memory_mb',
+                              'type': 'int'},
+                'old_state': {'fields': 'payload.old_state'},
+                'os_architecture': {
+                    'fields':
+                    "payload.image_meta.'org.openstack__1__architecture'"},
+                'os_distro': {
+                    'fields':
+                    "payload.image_meta.'org.openstack__1__os_distro'"},
+                'os_version': {
+                    'fields':
+                    "payload.image_meta.'org.openstack__1__os_version'"},
+                'resource_id': {'fields': 'payload.instance_id'},
+                'root_gb': {'fields': 'payload.root_gb',
+                            'type': 'int'},
+                'service': {'fields': 'publisher_id.`split(., 0, -1)`'},
+                'state': {'fields': 'payload.state'},
+                'tenant_id': {'fields': 'payload.tenant_id'},
+                'user_id': {'fields': 'payload.user_id'},
+                'vcpus': {'fields': 'payload.vcpus', 'type': 'int'}
+                }
+            }
+        config.append(instance_update)
+
     if 'maintenance.scheduled' in et_list:
-        add_mscheduled = False
         print('NOTE: maintenance.scheduled allready configured')
     else:
         print('NOTE: add maintenance.scheduled to event_definitions.yaml')
-        add_mscheduled = True
+        modify_config = True
         mscheduled = {
             'event_type': 'maintenance.scheduled',
             'traits': {
@@ -72,16 +129,15 @@ def set_maintenance_event_definitions():
                 'session_id': {'fields': 'payload.session_id'},
                 'project_id': {'fields': 'payload.project_id'},
                 'metadata': {'fields': 'payload.metadata'}
+                }
             }
-        }
         config.append(mscheduled)
 
     if 'maintenance.host' in et_list:
-        add_mhost = False
         print('NOTE: maintenance.host allready configured')
     else:
         print('NOTE: add maintenance.host to event_definitions.yaml')
-        add_mhost = True
+        modify_config = True
         mhost = {
             'event_type': 'maintenance.host',
             'traits': {
@@ -89,51 +145,18 @@ def set_maintenance_event_definitions():
                 'project_id': {'fields': 'payload.project_id'},
                 'state': {'fields': 'payload.state'},
                 'session_id': {'fields': 'payload.session_id'}
+                }
             }
-        }
         config.append(mhost)
 
-    if add_mscheduled or add_mhost:
-        shutil.copyfile(ed_file, ed_file_bak)
+    if modify_config:
+        if orig_ed_file_exist:
+            shutil.copyfile(ed_file, ed_file_bak)
+        else:
+            with open(ed_file_bak, 'w+') as file:
+                file.close()
         with open(ed_file, 'w+') as file:
             file.write(yaml.safe_dump(config))
 
-
-def set_cpu_allocation_ratio():
-    nova_file = '/etc/nova/nova.conf'
-    nova_file_bak = '/etc/nova/nova.bak'
-
-    if not os.path.isfile(nova_file):
-        raise Exception("File doesn't exist: %s." % nova_file)
-    # TODO (tojuvone): Unfortunately ConfigParser did not produce working conf
-    fcheck = open(nova_file)
-    found_list = ([ca for ca in fcheck.readlines() if "cpu_allocation_ratio"
-                  in ca])
-    fcheck.close()
-    if found_list and len(found_list):
-        change = False
-        found = False
-        for car in found_list:
-            if car.startswith('#'):
-                continue
-            if car.startswith('cpu_allocation_ratio'):
-                found = True
-                if "1.0" not in car.split('=')[1]:
-                    change = True
-    if not found or change:
-        # need to add or change
-        shutil.copyfile(nova_file, nova_file_bak)
-        fin = open(nova_file_bak)
-        fout = open(nova_file, "wt")
-        for line in fin:
-            if change and line.startswith("cpu_allocation_ratio"):
-                line = "cpu_allocation_ratio=1.0"
-            if not found and line.startswith("[DEFAULT]"):
-                line += "cpu_allocation_ratio=1.0\n"
-            fout.write(line)
-        fin.close()
-        fout.close()
-
 set_notifier_topic()
-set_maintenance_event_definitions()
-set_cpu_allocation_ratio()
+set_event_definitions()
diff --git a/doctor_tests/installer/common/set_congress.py b/doctor_tests/installer/common/set_congress.py
index d8438701..d2754838 100644
--- a/doctor_tests/installer/common/set_congress.py
+++ b/doctor_tests/installer/common/set_congress.py
@@ -7,12 +7,16 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 import configparser
+import os
 import shutil
 
 
 def set_drivers_config():
-    co_conf = "/etc/congress/congress.conf"
-    co_conf_bak = "/etc/congress/congress.conf.bak"
+    co_base = "/var/lib/config-data/puppet-generated/congress"
+    if not os.path.isdir(co_base):
+        co_base = ""
+    co_conf = co_base + "/etc/congress/congress.conf"
+    co_conf_bak = co_base + "/etc/congress/congress.conf.bak"
     doctor_driver = "congress.datasources.doctor_driver.DoctorDriver"
     config_modified = False
 
diff --git a/doctor_tests/installer/common/vitrage.py b/doctor_tests/installer/common/vitrage.py
index 30a73f5d..801adff5 100644
--- a/doctor_tests/installer/common/vitrage.py
+++ b/doctor_tests/installer/common/vitrage.py
@@ -9,8 +9,11 @@
 import os
 
 
+vi_base = "/var/lib/config-data/puppet-generated/vitrage"
+if not os.path.isdir(vi_base):
+    vi_base = ""
 vitrage_template_file = \
-    '/etc/vitrage/templates/vitrage_host_down_scenarios.yaml'
+    vi_base + '/etc/vitrage/templates/vitrage_host_down_scenarios.yaml'
 
 template = """
 metadata:
diff --git a/doctor_tests/scenario/maintenance.py b/doctor_tests/scenario/maintenance.py
index 54244d79..9fcd4128 100644
--- a/doctor_tests/scenario/maintenance.py
+++ b/doctor_tests/scenario/maintenance.py
@@ -168,9 +168,9 @@ class Maintenance(object):
         return ret.json()['state']
 
     def wait_maintenance_complete(self, session_id):
-        retries = 60
+        retries = 66
         state = None
-        time.sleep(600)
+        time.sleep(540)
         while state != 'MAINTENANCE_COMPLETE' and retries > 0:
             time.sleep(10)
             state = self.get_maintenance_state(session_id)
diff --git a/doctor_tests/stack.py b/doctor_tests/stack.py
index 688c2050..ee586fa8 100644
--- a/doctor_tests/stack.py
+++ b/doctor_tests/stack.py
@@ -44,7 +44,7 @@ class Stack(object):
         action_failed = '%s_FAILED' % action
 
         status = action_in_progress
-        stack_retries = 150
+        stack_retries = 160
         while status == action_in_progress and stack_retries > 0:
             time.sleep(2)
             try:
@@ -88,7 +88,19 @@ class Stack(object):
                                         template=template,
                                         parameters=parameters)
         self.stack_id = stack['stack']['id']
-        self.wait_stack_create()
+        try:
+            self.wait_stack_create()
+        except Exception:
+            # It might not always work at first
+            self.log.info('retry creating maintenance stack.......')
+            self.delete()
+            time.sleep(3)
+            stack = self.heat.stacks.create(stack_name=self.stack_name,
+                                            files=files,
+                                            template=template,
+                                            parameters=parameters)
+            self.stack_id = stack['stack']['id']
+            self.wait_stack_create()
 
     def update(self, stack_name, stack_id, template, parameters={}, files={}):
         self.heat.stacks.update(stack_name=stack_name,