Add setup and cleanup for maintenance test

JIRA: DOCTOR-106

Change-Id: Idb95e1391e8f9094f68e2c7bc79fd2d7e01af9de
Signed-off-by: Tomi Juvonen <tomi.juvonen@nokia.com>

14 files changed, 735 insertions, 91 deletions

diff --git a/doctor_tests/installer/apex.py b/doctor_tests/installer/apex.py
index c851b2ac..121767fc 100644
--- a/doctor_tests/installer/apex.py
+++ b/doctor_tests/installer/apex.py
@@ -6,14 +6,19 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
+import re
+import time
+
 from doctor_tests.common.utils import SSHClient
 from doctor_tests.installer.base import BaseInstaller
 
 
 class ApexInstaller(BaseInstaller):
     node_user_name = 'heat-admin'
-    cm_set_script = 'set_ceilometer.py'
-    cm_restore_script = 'restore_ceilometer.py'
+    cm_set_script = 'set_config.py'
+    cm_set_compute_script = 'set_compute_config.py'
+    cm_restore_script = 'restore_config.py'
+    cm_restore_compute_script = 'restore_compute_config.py'
 
     def __init__(self, conf, log):
         super(ApexInstaller, self).__init__(conf, log)
@@ -23,13 +28,15 @@ class ApexInstaller(BaseInstaller):
                                 look_for_keys=True)
         self.key_file = None
         self.controllers = list()
+        self.computes = list()
         self.controller_clients = list()
+        self.compute_clients = list()
 
     def setup(self):
         self.log.info('Setup Apex installer start......')
 
         self.key_file = self.get_ssh_key_from_installer()
-        self.controllers = self.get_controller_ips()
+        self._get_and_set_ips()
         self.create_flavor()
         self.set_apply_patches()
         self.setup_stunnel()
@@ -43,16 +50,20 @@ class ApexInstaller(BaseInstaller):
         key_path = '/home/stack/.ssh/id_rsa'
         return self._get_ssh_key(self.client, key_path)
 
-    def get_controller_ips(self):
-        self.log.info('Get controller ips from Apex installer......')
-
-        command = "source stackrc; " \
-                  "nova list | grep ' overcloud-controller-[0-9] ' " \
-                  "| sed -e 's/^.*ctlplane=//' |awk '{print $1}'"
-        controllers = self._run_cmd_remote(self.client, command)
-        self.log.info('Get controller_ips:%s from Apex installer'
-                      % controllers)
-        return controllers
+    def _get_and_set_ips(self):
+        self.log.info('Get controller and compute ips from Apex installer'
+                      '......')
+
+        command = "source stackrc; nova list | grep ' overcloud-'"
+        raw_ips_list = self._run_cmd_remote(self.client, command)
+        for line in raw_ips_list:
+            ip = line.split('ctlplane=', 1)[1].split(" ", 1)[0]
+            if 'overcloud-controller-' in line:
+                self.controllers.append(ip)
+            elif 'overcloud-novacompute-' in line:
+                self.computes.append(ip)
+        self.log.info('controller_ips:%s' % self.controllers)
+        self.log.info('compute_ips:%s' % self.computes)
 
     def get_host_ip_from_hostname(self, hostname):
         self.log.info('Get host ip by hostname=%s from Apex installer......'
@@ -63,12 +74,31 @@ class ApexInstaller(BaseInstaller):
         host_ips = self._run_cmd_remote(self.client, command)
         return host_ips[0]
 
+    def get_transport_url(self):
+        client = SSHClient(self.controllers[0], self.node_user_name,
+                           key_filename=self.key_file)
+
+        command = 'sudo grep "^transport_url" /etc/nova/nova.conf'
+        ret, url = client.ssh(command)
+        if ret:
+            raise Exception('Exec command to get host ip from controller(%s)'
+                            'in Apex installer failed, ret=%s, output=%s'
+                            % (self.controllers[0], ret, url))
+        # need to use ip instead of hostname
+        ret = (re.sub("@.*:", "@%s:" % self.controllers[0],
+               url[0].split("=", 1)[1]))
+        self.log.debug('get_transport_url %s' % ret)
+        return ret
+
     def set_apply_patches(self):
         self.log.info('Set apply patches start......')
 
         restart_cm_cmd = 'sudo systemctl restart ' \
                          'openstack-ceilometer-notification.service'
 
+        if self.conf.test_case != 'fault_management':
+            restart_cm_cmd += ' openstack-nova-scheduler.service'
+
         for node_ip in self.controllers:
             client = SSHClient(node_ip, self.node_user_name,
                                key_filename=self.key_file)
@@ -77,13 +107,38 @@ class ApexInstaller(BaseInstaller):
                                     restart_cm_cmd,
                                     self.cm_set_script)
 
+        if self.conf.test_case != 'fault_management':
+            restart_cm_cmd = 'sudo systemctl restart ' \
+                             'openstack-nova-compute.service'
+            for node_ip in self.computes:
+                client = SSHClient(node_ip, self.node_user_name,
+                                   key_filename=self.key_file)
+                self.compute_clients.append(client)
+                self._run_apply_patches(client,
+                                        restart_cm_cmd,
+                                        self.cm_set_compute_script)
+
+        if self.conf.test_case != 'fault_management':
+            time.sleep(10)
+
     def restore_apply_patches(self):
         self.log.info('restore apply patches start......')
 
         restart_cm_cmd = 'sudo systemctl restart ' \
                          'openstack-ceilometer-notification.service'
 
+        if self.conf.test_case != 'fault_management':
+            restart_cm_cmd += ' openstack-nova-scheduler.service'
+
         for client in self.controller_clients:
             self._run_apply_patches(client,
                                     restart_cm_cmd,
                                     self.cm_restore_script)
+
+        if self.conf.test_case != 'fault_management':
+            restart_cm_cmd = 'sudo systemctl restart ' \
+                             'openstack-nova-compute.service'
+            for client in self.compute_clients:
+                self._run_apply_patches(client,
+                                        restart_cm_cmd,
+                                        self.cm_restore_compute_script)
diff --git a/doctor_tests/installer/base.py b/doctor_tests/installer/base.py
index f37b7f1a..4eed3f29 100644
--- a/doctor_tests/installer/base.py
+++ b/doctor_tests/installer/base.py
@@ -58,21 +58,33 @@ class BaseInstaller(object):
     def setup_stunnel(self):
         self.log.info('Setup ssh stunnel in %s installer......'
                       % self.conf.installer.type)
+        tunnels = [self.conf.consumer.port]
+        if self.conf.test_case == 'maintenance':
+            tunnel_uptime = 1200
+            tunnels += [self.conf.app_manager.port, self.conf.inspector.port]
+        elif self.conf.test_case == 'all':
+            tunnel_uptime = 1800
+            tunnels += [self.conf.app_manager.port, self.conf.inspector.port]
+        else:
+            tunnel_uptime = 600
 
         for node_ip in self.controllers:
-            cmd = ("ssh -o UserKnownHostsFile=/dev/null"
-                   " -o StrictHostKeyChecking=no"
-                   " -i %s %s@%s -R %s:localhost:%s"
-                   " sleep 600 > ssh_tunnel.%s"
-                   " 2>&1 < /dev/null "
-                   % (self.key_file,
-                      self.node_user_name,
-                      node_ip,
-                      self.conf.consumer.port,
-                      self.conf.consumer.port,
-                      node_ip))
-            server = subprocess.Popen('exec ' + cmd, shell=True)
-            self.servers.append(server)
+            for port in tunnels:
+                self.log.info('tunnel for port %s' % port)
+                cmd = ("ssh -o UserKnownHostsFile=/dev/null"
+                       " -o StrictHostKeyChecking=no"
+                       " -i %s %s@%s -R %s:localhost:%s"
+                       " sleep %s > ssh_tunnel.%s"
+                       " 2>&1 < /dev/null "
+                       % (self.key_file,
+                          self.node_user_name,
+                          node_ip,
+                          port,
+                          port,
+                          tunnel_uptime,
+                          node_ip))
+                server = subprocess.Popen('exec ' + cmd, shell=True)
+                self.servers.append(server)
 
     def _get_ssh_key(self, client, key_path):
         self.log.info('Get SSH keys from %s installer......'
diff --git a/doctor_tests/installer/common/restore_compute_config.py b/doctor_tests/installer/common/restore_compute_config.py
new file mode 100644
index 00000000..0971d12b
--- /dev/null
+++ b/doctor_tests/installer/common/restore_compute_config.py
@@ -0,0 +1,25 @@
+##############################################################################
+# Copyright (c) 2018 Nokia Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+
+
+def restore_cpu_allocation_ratio():
+    nova_file = '/etc/nova/nova.conf'
+    nova_file_bak = '/etc/nova/nova.bak'
+
+    if not os.path.isfile(nova_file_bak):
+        print('Bak_file:%s does not exist.' % nova_file_bak)
+    else:
+        print('restore: %s' % nova_file)
+        shutil.copyfile(nova_file_bak, nova_file)
+        os.remove(nova_file_bak)
+    return
+
+restore_cpu_allocation_ratio()
diff --git a/doctor_tests/installer/common/restore_ceilometer.py b/doctor_tests/installer/common/restore_config.py
index d25b9ede..c1f919c1 100644
--- a/doctor_tests/installer/common/restore_ceilometer.py
+++ b/doctor_tests/installer/common/restore_config.py
@@ -24,4 +24,32 @@ def restore_ep_config():
     return
 
 
+def restore_ed_config():
+
+    ed_file = '/etc/ceilometer/event_definitions.yaml'
+    ed_file_bak = '/etc/ceilometer/event_definitions.bak'
+
+    if not os.path.isfile(ed_file_bak):
+        print("Bak_file doesn't exist: %s." % ed_file_bak)
+    else:
+        print('restore: %s' % ed_file)
+        shutil.copyfile(ed_file_bak, ed_file)
+        os.remove(ed_file_bak)
+    return
+
+
+def restore_cpu_allocation_ratio():
+    nova_file = '/etc/nova/nova.conf'
+    nova_file_bak = '/etc/nova/nova.bak'
+
+    if not os.path.isfile(nova_file_bak):
+        print('Bak_file:%s does not exist.' % nova_file_bak)
+    else:
+        print('restore: %s' % nova_file)
+        shutil.copyfile(nova_file_bak, nova_file)
+        os.remove(nova_file_bak)
+    return
+
 restore_ep_config()
+restore_ed_config()
+restore_cpu_allocation_ratio()
diff --git a/doctor_tests/installer/common/set_ceilometer.py b/doctor_tests/installer/common/set_ceilometer.py
deleted file mode 100644
index 4050aaef..00000000
--- a/doctor_tests/installer/common/set_ceilometer.py
+++ /dev/null
@@ -1,45 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-import os
-import shutil
-import yaml
-
-ep_file = '/etc/ceilometer/event_pipeline.yaml'
-ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
-event_notifier_topic = 'notifier://?topic=alarm.all'
-
-
-def set_notifier_topic():
-    config_modified = False
-
-    if not os.path.isfile(ep_file):
-        raise Exception("File doesn't exist: %s." % ep_file)
-
-    with open(ep_file, 'r') as file:
-        config = yaml.safe_load(file)
-
-    sinks = config['sinks']
-    for sink in sinks:
-        if sink['name'] == 'event_sink':
-            publishers = sink['publishers']
-            if event_notifier_topic not in publishers:
-                print('Add event notifier in ceilometer')
-                publishers.append(event_notifier_topic)
-                config_modified = True
-            else:
-                print('NOTE: event notifier is configured'
-                      'in ceilometer as we needed')
-
-    if config_modified:
-        shutil.copyfile(ep_file, ep_file_bak)
-        with open(ep_file, 'w+') as file:
-            file.write(yaml.safe_dump(config))
-
-
-set_notifier_topic()
diff --git a/doctor_tests/installer/common/set_compute_config.py b/doctor_tests/installer/common/set_compute_config.py
new file mode 100644
index 00000000..07db1e16
--- /dev/null
+++ b/doctor_tests/installer/common/set_compute_config.py
@@ -0,0 +1,48 @@
+##############################################################################
+# Copyright (c) 2018 Nokia Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+
+
+def set_cpu_allocation_ratio():
+    nova_file = '/etc/nova/nova.conf'
+    nova_file_bak = '/etc/nova/nova.bak'
+
+    if not os.path.isfile(nova_file):
+        raise Exception("File doesn't exist: %s." % nova_file)
+    # TODO (tojuvone): Unfortunately ConfigParser did not produce working conf
+    fcheck = open(nova_file)
+    found_list = ([ca for ca in fcheck.readlines() if "cpu_allocation_ratio"
+                  in ca])
+    fcheck.close()
+    if found_list and len(found_list):
+        change = False
+        found = False
+        for car in found_list:
+            if car.startswith('#'):
+                continue
+            if car.startswith('cpu_allocation_ratio'):
+                found = True
+                if "1.0" not in car.split('=')[1]:
+                    change = True
+    if not found or change:
+        # need to add or change
+        shutil.copyfile(nova_file, nova_file_bak)
+        fin = open(nova_file_bak)
+        fout = open(nova_file, "wt")
+        for line in fin:
+            if change and line.startswith("cpu_allocation_ratio"):
+                line = "cpu_allocation_ratio=1.0"
+            if not found and line.startswith("[DEFAULT]"):
+                line += "cpu_allocation_ratio=1.0\n"
+            fout.write(line)
+        fin.close()
+        fout.close()
+
+set_cpu_allocation_ratio()
diff --git a/doctor_tests/installer/common/set_config.py b/doctor_tests/installer/common/set_config.py
new file mode 100644
index 00000000..42465247
--- /dev/null
+++ b/doctor_tests/installer/common/set_config.py
@@ -0,0 +1,139 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import shutil
+import yaml
+
+ep_file = '/etc/ceilometer/event_pipeline.yaml'
+ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak'
+event_notifier_topic = 'notifier://?topic=alarm.all'
+
+
+def set_notifier_topic():
+    config_modified = False
+
+    if not os.path.isfile(ep_file):
+        raise Exception("File doesn't exist: %s." % ep_file)
+
+    with open(ep_file, 'r') as file:
+        config = yaml.safe_load(file)
+
+    sinks = config['sinks']
+    for sink in sinks:
+        if sink['name'] == 'event_sink':
+            publishers = sink['publishers']
+            if event_notifier_topic not in publishers:
+                print('Add event notifier in ceilometer')
+                publishers.append(event_notifier_topic)
+                config_modified = True
+            else:
+                print('NOTE: event notifier is configured'
+                      'in ceilometer as we needed')
+
+    if config_modified:
+        shutil.copyfile(ep_file, ep_file_bak)
+        with open(ep_file, 'w+') as file:
+            file.write(yaml.safe_dump(config))
+
+
+def set_maintenance_event_definitions():
+    ed_file = '/etc/ceilometer/event_definitions.yaml'
+    ed_file_bak = '/etc/ceilometer/event_definitions.bak'
+
+    if not os.path.isfile(ed_file):
+        raise Exception("File doesn't exist: %s." % ed_file)
+
+    with open(ed_file, 'r') as file:
+        config = yaml.safe_load(file)
+
+    et_list = [et['event_type'] for et in config]
+
+    if 'maintenance.scheduled' in et_list:
+        add_mscheduled = False
+        print('NOTE: maintenance.scheduled allready configured')
+    else:
+        print('NOTE: add maintenance.scheduled to event_definitions.yaml')
+        add_mscheduled = True
+        mscheduled = {
+            'event_type': 'maintenance.scheduled',
+            'traits': {
+                'allowed_actions': {'fields': 'payload.allowed_actions'},
+                'instance_ids': {'fields': 'payload.instance_ids'},
+                'reply_url': {'fields': 'payload.reply_url'},
+                'actions_at': {'fields': 'payload.actions_at',
+                               'type': 'datetime'},
+                'state': {'fields': 'payload.state'},
+                'session_id': {'fields': 'payload.session_id'},
+                'project_id': {'fields': 'payload.project_id'},
+                'metadata': {'fields': 'payload.metadata'}
+            }
+        }
+        config.append(mscheduled)
+
+    if 'maintenance.host' in et_list:
+        add_mhost = False
+        print('NOTE: maintenance.host allready configured')
+    else:
+        print('NOTE: add maintenance.host to event_definitions.yaml')
+        add_mhost = True
+        mhost = {
+            'event_type': 'maintenance.host',
+            'traits': {
+                'host': {'fields': 'payload.host'},
+                'project_id': {'fields': 'payload.project_id'},
+                'state': {'fields': 'payload.state'},
+                'session_id': {'fields': 'payload.session_id'}
+            }
+        }
+        config.append(mhost)
+
+    if add_mscheduled or add_mhost:
+        shutil.copyfile(ed_file, ed_file_bak)
+        with open(ed_file, 'w+') as file:
+            file.write(yaml.safe_dump(config))
+
+
+def set_cpu_allocation_ratio():
+    nova_file = '/etc/nova/nova.conf'
+    nova_file_bak = '/etc/nova/nova.bak'
+
+    if not os.path.isfile(nova_file):
+        raise Exception("File doesn't exist: %s." % nova_file)
+    # TODO (tojuvone): Unfortunately ConfigParser did not produce working conf
+    fcheck = open(nova_file)
+    found_list = ([ca for ca in fcheck.readlines() if "cpu_allocation_ratio"
+                  in ca])
+    fcheck.close()
+    if found_list and len(found_list):
+        change = False
+        found = False
+        for car in found_list:
+            if car.startswith('#'):
+                continue
+            if car.startswith('cpu_allocation_ratio'):
+                found = True
+                if "1.0" not in car.split('=')[1]:
+                    change = True
+    if not found or change:
+        # need to add or change
+        shutil.copyfile(nova_file, nova_file_bak)
+        fin = open(nova_file_bak)
+        fout = open(nova_file, "wt")
+        for line in fin:
+            if change and line.startswith("cpu_allocation_ratio"):
+                line = "cpu_allocation_ratio=1.0"
+            if not found and line.startswith("[DEFAULT]"):
+                line += "cpu_allocation_ratio=1.0\n"
+            fout.write(line)
+        fin.close()
+        fout.close()
+
+set_notifier_topic()
+set_maintenance_event_definitions()
+set_cpu_allocation_ratio()
diff --git a/doctor_tests/main.py b/doctor_tests/main.py
index 79480ca1..2a8abda7 100644
--- a/doctor_tests/main.py
+++ b/doctor_tests/main.py
@@ -10,6 +10,7 @@ import os
 from os.path import isfile, join
 import sys
 import time
+from traceback import format_exc
 
 from doctor_tests import config
 from doctor_tests.identity_auth import get_identity_auth
@@ -17,8 +18,9 @@ from doctor_tests.identity_auth import get_session
 from doctor_tests.image import Image
 from doctor_tests.installer import get_installer
 import doctor_tests.logger as doctor_log
-from doctor_tests.os_clients import nova_client
 from doctor_tests.scenario.fault_management import FaultManagement
+from doctor_tests.os_clients import nova_client
+from doctor_tests.scenario.maintenance import Maintenance
 from doctor_tests.user import User
 
 
@@ -92,20 +94,33 @@ class DoctorTest(object):
             LOG.info('not enough compute nodes, skipping doctor '
                      'maintenance test')
             return
+        elif self.conf.installer.type != 'apex':
+            LOG.info('not supported installer, skipping doctor '
+                     'maintenance test')
+            return
         try:
             LOG.info('doctor maintenance test starting.......')
-            # TODO (tojuvone) test setup and actual test
+
+            maintenance = Maintenance(self.conf, LOG)
+            maintenance.setup_maintenance(self.user)
+
+            # TODO (tojuvone) actual test
+
         except Exception as e:
             LOG.error('doctor maintenance test failed, Exception=%s' % e)
+            LOG.error(format_exc())
             sys.exit(1)
-        # TODO (tojuvone) finally: test case specific cleanup
+        finally:
+            maintenance.cleanup_maintenance()
 
     def run(self):
         """run doctor tests"""
         try:
             LOG.info('doctor test starting.......')
+
             # prepare common test env
             self.setup()
+
             if self.conf.test_case == 'all':
                 self.test_fault_management()
                 self.test_maintenance()
diff --git a/doctor_tests/maintenance_hot_tpl.yaml b/doctor_tests/maintenance_hot_tpl.yaml
new file mode 100644
index 00000000..e2e47023
--- /dev/null
+++ b/doctor_tests/maintenance_hot_tpl.yaml
@@ -0,0 +1,119 @@
+---
+heat_template_version: 2017-02-24
+description: Doctor Maintenance test case
+
+parameters:
+  ext_net:
+    type: string
+    default: external
+  flavor_vcpus:
+    type: number
+    default: 24
+  maint_image:
+    type: string
+    default: cirros
+  ha_intances:
+    type: number
+    default: 2
+  nonha_intances:
+    type: number
+    default: 4
+  app_manager_alarm_url:
+    type: string
+    default: http://0.0.0.0:12348/maintenance
+  inpector_alarm_url:
+    type: string
+    default: http://0.0.0.0:12345/maintenance
+
+
+resources:
+  int_net:
+    type: OS::Neutron::Net
+
+  int_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: {get_resource: int_net}
+      cidr: "9.9.9.0/24"
+      dns_nameservers: ["8.8.8.8"]
+      ip_version: 4
+
+  int_router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info: {network: {get_param: ext_net}}
+
+  int_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: {get_resource: int_router}
+      subnet: {get_resource: int_subnet}
+
+  maint_instance_flavor:
+    type: OS::Nova::Flavor
+    properties:
+      name: doctor_maint_flavor
+      ram: 512
+      vcpus: {get_param: flavor_vcpus}
+      disk: 1
+
+  ha_app_svrgrp:
+    type: OS::Nova::ServerGroup
+    properties:
+      name: doctor_ha_app_group
+      policies: ['anti-affinity']
+
+  floating_ip:
+    type: OS::Nova::FloatingIP
+    properties:
+      pool: {get_param: ext_net}
+
+  multi_ha_instances:
+    type: OS::Heat::ResourceGroup
+    properties:
+      count: {get_param: ha_intances}
+      resource_def:
+        type: OS::Nova::Server
+        properties:
+          name: doctor_ha_app_%index%
+          flavor: {get_resource: maint_instance_flavor}
+          image: {get_param: maint_image}
+          networks:
+            - network: {get_resource: int_net}
+          scheduler_hints:
+            group: {get_resource: ha_app_svrgrp}
+
+  multi_nonha_instances:
+    type: OS::Heat::ResourceGroup
+    properties:
+      count: {get_param: nonha_intances}
+      resource_def:
+        type: OS::Nova::Server
+        properties:
+          name: doctor_nonha_app_%index%
+          flavor: {get_resource: maint_instance_flavor}
+          image: {get_param: maint_image}
+          networks:
+            - network: {get_resource: int_net}
+
+  association:
+    type: OS::Nova::FloatingIPAssociation
+    properties:
+      floating_ip: {get_resource: floating_ip}
+      server_id: {get_attr: [multi_ha_instances, resource.0]}
+
+  app_manager_alarm:
+    type: OS::Aodh::EventAlarm
+    properties:
+      alarm_actions:
+        - {get_param: app_manager_alarm_url}
+      event_type: "maintenance.scheduled"
+      repeat_actions: true
+
+  inpector_alarm:
+    type: OS::Aodh::EventAlarm
+    properties:
+      alarm_actions:
+        - {get_param: inpector_alarm_url}
+      event_type: "maintenance.host"
+      repeat_actions: true
diff --git a/doctor_tests/os_clients.py b/doctor_tests/os_clients.py
index 640281df..7ab4e9b4 100644
--- a/doctor_tests/os_clients.py
+++ b/doctor_tests/os_clients.py
@@ -11,6 +11,7 @@ from oslo_config import cfg
 import aodhclient.client as aodhclient

 from congressclient.v1 import client as congressclient

 import glanceclient.client as glanceclient

+import heatclient.client as heatclient

 from keystoneclient import client as ks_client

 from neutronclient.v2_0 import client as neutronclient

 import novaclient.client as novaclient

@@ -23,6 +24,7 @@ OPTS = [
     cfg.StrOpt('aodh_version', default='2', help='aodh version'),

     cfg.StrOpt('vitrage_version', default='1', help='vitrage version'),

     cfg.StrOpt('keystone_version', default='v3', help='keystone version'),

+    cfg.StrOpt('heat_version', default='1', help='heat version'),

 ]

 

 

@@ -31,6 +33,11 @@ def glance_client(version, session):
                                session=session)

 

 

+def heat_client(version, session):

+    return heatclient.Client(version=version,

+                             session=session)

+

+

 def keystone_client(version, session):

     return ks_client.Client(version=version,

                             session=session)

diff --git a/doctor_tests/scenario/maintenance.py b/doctor_tests/scenario/maintenance.py
new file mode 100644
index 00000000..bb0e943b
--- /dev/null
+++ b/doctor_tests/scenario/maintenance.py
@@ -0,0 +1,103 @@
+##############################################################################
+# Copyright (c) 2018 Nokia Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from doctor_tests.common.utils import get_doctor_test_root_dir
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import keystone_client
+from doctor_tests.os_clients import neutron_client
+from doctor_tests.os_clients import nova_client
+from doctor_tests.stack import Stack
+
+
+class Maintenance(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+        self.keystone = keystone_client(
+            self.conf.keystone_version, get_session())
+        self.nova = nova_client(conf.nova_version, get_session())
+        auth = get_identity_auth(project=self.conf.doctor_project)
+        self.neutron = neutron_client(get_session(auth=auth))
+        self.stack = Stack(self.conf, self.log)
+
+    def get_external_network(self):
+        ext_net = None
+        networks = self.neutron.list_networks()['networks']
+        for network in networks:
+            if network['router:external']:
+                ext_net = network['name']
+                break
+        if ext_net is None:
+            raise Exception("externl network not defined")
+        return ext_net
+
+    def setup_maintenance(self, user):
+        # each hypervisor needs to have same amount of vcpus and they
+        # need to be free before test
+        hvisors = self.nova.hypervisors.list(detailed=True)
+        prev_vcpus = 0
+        prev_hostname = ""
+        self.log.info('checking hypervisors.......')
+        for hvisor in hvisors:
+            vcpus = hvisor.__getattr__("vcpus")
+            vcpus_used = hvisor.__getattr__("vcpus_used")
+            hostname = hvisor.__getattr__("hypervisor_hostname")
+            if vcpus < 2:
+                raise Exception('not enough vcpus on %s' % hostname)
+            if vcpus_used > 0:
+                raise Exception('%d vcpus used on %s'
+                                % (vcpus_used, hostname))
+            if prev_vcpus != 0 and prev_vcpus != vcpus:
+                raise Exception('%d vcpus on %s does not match to'
+                                '%d on %s'
+                                % (vcpus, hostname,
+                                   prev_vcpus, prev_hostname))
+            prev_vcpus = vcpus
+            prev_hostname = hostname
+
+        # maintenance flavor made so that 2 instances take whole node
+        flavor_vcpus = int(vcpus / 2)
+        compute_nodes = len(hvisors)
+        amount_actstdby_instances = 2
+        amount_noredundancy_instances = 2 * compute_nodes - 2
+        self.log.info('testing %d computes with %d vcpus each'
+                      % (compute_nodes, vcpus))
+        self.log.info('testing %d actstdby and %d noredundancy instances'
+                      % (amount_actstdby_instances,
+                         amount_noredundancy_instances))
+        max_instances = (amount_actstdby_instances +
+                         amount_noredundancy_instances)
+        max_cores = compute_nodes * vcpus
+
+        user.update_quota(max_instances, max_cores)
+
+        test_dir = get_doctor_test_root_dir()
+        template_file = '{0}/{1}'.format(test_dir, 'maintenance_hot_tpl.yaml')
+        files, template = self.stack.get_hot_tpl(template_file)
+
+        ext_net = self.get_external_network()
+
+        parameters = {'ext_net': ext_net,
+                      'flavor_vcpus': flavor_vcpus,
+                      'maint_image': self.conf.image_name,
+                      'nonha_intances': amount_noredundancy_instances,
+                      'ha_intances': amount_actstdby_instances}
+
+        self.log.info('creating maintenance stack.......')
+        self.log.info('parameters: %s' % parameters)
+
+        self.stack.create('doctor_test_maintenance',
+                          template,
+                          parameters=parameters,
+                          files=files)
+
+    def cleanup_maintenance(self):
+        self.log.info('stack delete start.......')
+        self.stack.delete()
diff --git a/doctor_tests/stack.py b/doctor_tests/stack.py
new file mode 100644
index 00000000..688c2050
--- /dev/null
+++ b/doctor_tests/stack.py
@@ -0,0 +1,106 @@
+##############################################################################
+# Copyright (c) 2018 Nokia Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import time
+
+from heatclient.common.template_utils import get_template_contents
+from heatclient import exc as heat_excecption
+
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.os_clients import heat_client
+
+
+class Stack(object):
+
+    def __init__(self, conf, log):
+        self.conf = conf
+        self.log = log
+        auth = get_identity_auth(project=self.conf.doctor_project)
+        self.heat = heat_client(self.conf.heat_version,
+                                get_session(auth=auth))
+        self.stack_name = None
+        self.stack_id = None
+        self.template = None
+        self.parameters = {}
+        self.files = {}
+
+    # standard yaml.load will not work for hot tpl becasue of date format in
+    # heat_template_version is not string
+    def get_hot_tpl(self, template_file):
+        if not os.path.isfile(template_file):
+            raise Exception('File(%s) does not exist' % template_file)
+        return get_template_contents(template_file=template_file)
+
+    def _wait_stack_action_complete(self, action):
+        action_in_progress = '%s_IN_PROGRESS' % action
+        action_complete = '%s_COMPLETE' % action
+        action_failed = '%s_FAILED' % action
+
+        status = action_in_progress
+        stack_retries = 150
+        while status == action_in_progress and stack_retries > 0:
+            time.sleep(2)
+            try:
+                stack = self.heat.stacks.get(self.stack_name)
+            except heat_excecption.HTTPNotFound:
+                if action == 'DELETE':
+                    # Might happen you never get status as stack deleted
+                    status = action_complete
+                    break
+                else:
+                    raise Exception('unable to get stack')
+            status = stack.stack_status
+            stack_retries = stack_retries - 1
+        if stack_retries == 0 and status != action_complete:
+            raise Exception("stack %s not completed within 5min, status:"
+                            " %s" % (action, status))
+        elif status == action_complete:
+            self.log.info('stack %s %s' % (self.stack_name, status))
+        elif status == action_failed:
+            raise Exception("stack %s failed" % action)
+        else:
+            self.log.error('stack %s %s' % (self.stack_name, status))
+            raise Exception("stack %s unknown result" % action)
+
+    def wait_stack_delete(self):
+        self._wait_stack_action_complete('DELETE')
+
+    def wait_stack_create(self):
+        self._wait_stack_action_complete('CREATE')
+
+    def wait_stack_update(self):
+        self._wait_stack_action_complete('UPDATE')
+
+    def create(self, stack_name, template, parameters={}, files={}):
+        self.stack_name = stack_name
+        self.template = template
+        self.parameters = parameters
+        self.files = files
+        stack = self.heat.stacks.create(stack_name=self.stack_name,
+                                        files=files,
+                                        template=template,
+                                        parameters=parameters)
+        self.stack_id = stack['stack']['id']
+        self.wait_stack_create()
+
+    def update(self, stack_name, stack_id, template, parameters={}, files={}):
+        self.heat.stacks.update(stack_name=stack_name,
+                                stack_id=stack_id,
+                                files=files,
+                                template=template,
+                                parameters=parameters)
+        self.wait_stack_update()
+
+    def delete(self):
+        if self.stack_id is not None:
+            self.heat.stacks.delete(self.stack_name)
+            self.wait_stack_delete()
+        else:
+            self.log.info('no stack to delete')
diff --git a/doctor_tests/user.py b/doctor_tests/user.py
index fee3e1fb..29aa004b 100644
--- a/doctor_tests/user.py
+++ b/doctor_tests/user.py
@@ -8,12 +8,12 @@
 ##############################################################################
 import os
 
+from keystoneclient import exceptions as ks_exceptions
 from oslo_config import cfg
 
 from doctor_tests.identity_auth import get_session
 from doctor_tests.os_clients import keystone_client
 from doctor_tests.os_clients import nova_client
-from keystoneclient import exceptions as ks_exceptions
 
 
 OPTS = [
@@ -53,10 +53,11 @@ class User(object):
     def __init__(self, conf, log):
         self.conf = conf
         self.log = log
+        self.def_quota = None
+        self.restore_def_quota = False
         self.keystone = keystone_client(
             self.conf.keystone_version, get_session())
-        self.nova = \
-            nova_client(conf.nova_version, get_session())
+        self.nova = nova_client(conf.nova_version, get_session())
         self.users = {}
         self.projects = {}
         self.roles = {}
@@ -83,10 +84,9 @@ class User(object):
                              domain=self.conf.doctor_domain_id)}
         if self.conf.doctor_project not in self.projects:
             self.log.info('create project......')
-            test_project = \
-                self.keystone.projects.create(
-                    self.conf.doctor_project,
-                    self.conf.doctor_domain_id)
+            test_project = self.keystone.projects.create(
+                self.conf.doctor_project,
+                self.conf.doctor_domain_id)
             self.projects[test_project.name] = test_project
         else:
             self.log.info('project %s already created......'
@@ -151,6 +151,13 @@ class User(object):
             self.keystone.roles.grant(role, user=user, project=project)
             roles_for_user[role_name] = role
 
+    def _restore_default_quota(self):
+        if self.def_quota is not None and self.restore_def_quota:
+            self.log.info('restore default quota......')
+            self.nova.quota_classes.update('default',
+                                           instances=self.def_quota.instances,
+                                           cores=self.def_quota.cores)
+
     def delete(self):
         """delete the test user, project and role"""
         self.log.info('user delete start......')
@@ -159,6 +166,8 @@ class User(object):
         user = self.users.get(self.conf.doctor_user)
         role = self.roles.get(self.conf.doctor_role)
 
+        self._restore_default_quota()
+
         if project:
             if 'admin' in self.roles_for_admin:
                 self.keystone.roles.revoke(
@@ -177,23 +186,45 @@ class User(object):
             self.keystone.projects.delete(project)
         self.log.info('user delete end......')
 
-    def update_quota(self):
-        self.log.info('user quota update start......')
+    def update_quota(self, instances=None, cores=None):
+        self.log.info('quota update start......')
         project = self.projects.get(self.conf.doctor_project)
+
         user = self.users.get(self.conf.doctor_user)
 
+        if instances is not None:
+            quota_instances = instances
+        else:
+            quota_instances = self.conf.quota_instances
+        if cores is not None:
+            quota_cores = cores
+        else:
+            quota_cores = self.conf.quota_cores
+
         if project and user:
+            # default needs to be at least the same as with doctor_user
+            self.log.info('default quota update start......')
+
+            self.def_quota = self.nova.quota_classes.get('default')
+            if quota_instances > self.def_quota.instances:
+                self.restore_def_quota = True
+                self.nova.quota_classes.update('default',
+                                               instances=quota_instances)
+            if quota_cores > self.def_quota.cores:
+                self.restore_def_quota = True
+                self.nova.quota_classes.update('default',
+                                               cores=quota_cores)
+            self.log.info('user quota update start......')
             self.quota = self.nova.quotas.get(project.id,
                                               user_id=user.id)
-            if self.conf.quota_instances > self.quota.instances:
-                self.nova.quotas.update(
-                    project.id,
-                    instances=self.conf.quota_instances,
-                    user_id=user.id)
-            if self.conf.quota_cores > self.quota.cores:
+            if quota_instances > self.quota.instances:
+                self.nova.quotas.update(project.id,
+                                        instances=quota_instances,
+                                        user_id=user.id)
+            if quota_cores > self.quota.cores:
                 self.nova.quotas.update(project.id,
-                                        cores=self.conf.quota_cores,
+                                        cores=quota_cores,
                                         user_id=user.id)
-            self.log.info('user quota update end......')
         else:
             raise Exception('No project or role for update quota')
+        self.log.info('quota update end......')
diff --git a/requirements.txt b/requirements.txt
index b60878fc..4d2fffff 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,4 +16,5 @@ python-congressclient<2000,>=1.9.0 # Apache-2.0
 python-glanceclient>=2.8.0 # Apache-2.0
 python-vitrageclient>=2.0.0 # Apache-2.0
 virtualenv>=14.0.6 # MIT
+python-heatclient>=1.8.2 # Apache-2.0
 flake8<2.6.0,>=2.5.4 # MIT