diff options
Diffstat (limited to 'doctor_tests')
| -rw-r--r-- | doctor_tests/common/utils.py | 4 | ||||
| -rw-r--r-- | doctor_tests/installer/apex.py | 99 | ||||
| -rw-r--r-- | doctor_tests/installer/base.py | 20 | ||||
| -rw-r--r-- | doctor_tests/installer/common/restart_aodh.py | 42 | ||||
| -rw-r--r-- | doctor_tests/installer/common/restore_aodh.py | 32 | ||||
| -rw-r--r-- | doctor_tests/installer/common/restore_compute_config.py | 7 | ||||
| -rw-r--r-- | doctor_tests/installer/common/restore_config.py | 33 | ||||
| -rw-r--r-- | doctor_tests/installer/common/restore_congress.py | 7 | ||||
| -rw-r--r-- | doctor_tests/installer/common/set_compute_config.py | 21 | ||||
| -rw-r--r-- | doctor_tests/installer/common/set_config.py | 135 | ||||
| -rw-r--r-- | doctor_tests/installer/common/set_congress.py | 8 | ||||
| -rw-r--r-- | doctor_tests/installer/common/vitrage.py | 5 | ||||
| -rw-r--r-- | doctor_tests/scenario/maintenance.py | 4 | ||||
| -rw-r--r-- | doctor_tests/stack.py | 16 |
14 files changed, 316 insertions, 117 deletions
diff --git a/doctor_tests/common/utils.py b/doctor_tests/common/utils.py index 1a84c824..1a8840dd 100644 --- a/doctor_tests/common/utils.py +++ b/doctor_tests/common/utils.py @@ -67,7 +67,7 @@ class SSHClient(object): def __del__(self): self.client.close() - def ssh(self, command): + def ssh(self, command, raise_enabled=True): if self.log: self.log.info("Executing: %s" % command) stdin, stdout, stderr = self.client.exec_command(command) @@ -75,7 +75,7 @@ class SSHClient(object): output = list() for line in stdout.read().splitlines(): output.append(line.decode('utf-8')) - if ret: + if ret and raise_enabled: if self.log: self.log.info("*** FAILED to run command %s (%s)" % (command, ret)) diff --git a/doctor_tests/installer/apex.py b/doctor_tests/installer/apex.py index 694adb88..9b0010e4 100644 --- a/doctor_tests/installer/apex.py +++ b/doctor_tests/installer/apex.py @@ -16,17 +16,21 @@ from doctor_tests.installer.base import BaseInstaller class ApexInstaller(BaseInstaller): node_user_name = 'heat-admin' + installer_username = 'stack' cm_set_script = 'set_config.py' nc_set_compute_script = 'set_compute_config.py' cg_set_script = 'set_congress.py' cm_restore_script = 'restore_config.py' nc_restore_compute_script = 'restore_compute_config.py' cg_restore_script = 'restore_congress.py' + ac_restart_script = 'restart_aodh.py' + ac_restore_script = 'restore_aodh.py' + python = 'python' def __init__(self, conf, log): super(ApexInstaller, self).__init__(conf, log) self.client = SSHClient(self.conf.installer.ip, - self.conf.installer.username, + self.installer_username, key_filename=self.conf.installer.key_file, look_for_keys=True) self.key_file = None @@ -38,7 +42,7 @@ class ApexInstaller(BaseInstaller): def setup(self): self.log.info('Setup Apex installer start......') self.key_file = self.get_ssh_key_from_installer() - self._get_and_set_ips() + self._get_overcloud_conf() self.create_flavor() self.set_apply_patches() self.setup_stunnel() @@ -52,8 +56,8 @@ class ApexInstaller(BaseInstaller): key_path = '/home/stack/.ssh/id_rsa' return self._get_ssh_key(self.client, key_path) - def _get_and_set_ips(self): - self.log.info('Get controller and compute ips from Apex installer' + def _get_overcloud_conf(self): + self.log.info('Get overcloud config details from Apex installer' '......') command = "source stackrc; nova list | grep ' overcloud-'" @@ -64,8 +68,11 @@ class ApexInstaller(BaseInstaller): self.controllers.append(ip) elif 'overcloud-novacompute-' in line: self.computes.append(ip) + command = "grep docker /home/stack/deploy_command" + self.use_containers = self._check_cmd_remote(self.client, command) self.log.info('controller_ips:%s' % self.controllers) self.log.info('compute_ips:%s' % self.computes) + self.log.info('use_containers:%s' % self.use_containers) def get_host_ip_from_hostname(self, hostname): self.log.info('Get host ip by hostname=%s from Apex installer......' @@ -79,8 +86,12 @@ class ApexInstaller(BaseInstaller): def get_transport_url(self): client = SSHClient(self.controllers[0], self.node_user_name, key_filename=self.key_file) + if self.use_containers: + ncbase = "/var/lib/config-data/puppet-generated/nova" + else: + ncbase = "" + command = 'sudo grep "^transport_url" %s/etc/nova/nova.conf' % ncbase - command = 'sudo grep "^transport_url" /etc/nova/nova.conf' ret, url = client.ssh(command) if ret: raise Exception('Exec command to get host ip from controller(%s)' @@ -92,19 +103,39 @@ class ApexInstaller(BaseInstaller): self.log.debug('get_transport_url %s' % ret) return ret + def _set_docker_restart_cmd(self, service): + # There can be multiple instances running so need to restart all + cmd = "for container in `sudo docker ps | grep " + cmd += service + cmd += " | awk '{print $1}'`; do sudo docker restart $container; \ + done;" + return cmd + def set_apply_patches(self): self.log.info('Set apply patches start......') - restart_cmd = 'sudo systemctl restart' \ - ' openstack-ceilometer-notification.service' - set_scripts = [self.cm_set_script] + if self.use_containers: + restart_cmd = (self._set_docker_restart_cmd( + "ceilometer-notification")) + set_scripts.append(self.ac_restart_script) + else: + restart_cmd = 'sudo systemctl restart' \ + ' openstack-ceilometer-notification.service' + if self.conf.test_case != 'fault_management': - restart_cmd += ' openstack-nova-scheduler.service' + if self.use_containers: + restart_cmd += self._set_docker_restart_cmd("nova-scheduler") + else: + restart_cmd += ' openstack-nova-scheduler.service' + set_scripts.append(self.nc_set_compute_script) if self.conf.inspector.type == Inspector.CONGRESS: - restart_cmd += ' openstack-congress-server.service' + if self.use_containers: + restart_cmd += self._set_docker_restart_cmd("congress-server") + else: + restart_cmd += ' openstack-congress-server.service' set_scripts.append(self.cg_set_script) for node_ip in self.controllers: @@ -113,18 +144,23 @@ class ApexInstaller(BaseInstaller): self.controller_clients.append(client) self._run_apply_patches(client, restart_cmd, - set_scripts) + set_scripts, + python=self.python) if self.conf.test_case != 'fault_management': - restart_cmd = 'sudo systemctl restart' \ - ' openstack-nova-compute.service' + if self.use_containers: + restart_cmd = self._set_docker_restart_cmd("nova-compute") + else: + restart_cmd = 'sudo systemctl restart' \ + ' openstack-nova-compute.service' for node_ip in self.computes: client = SSHClient(node_ip, self.node_user_name, key_filename=self.key_file) self.compute_clients.append(client) self._run_apply_patches(client, restart_cmd, - [self.nc_set_compute_script]) + [self.nc_set_compute_script], + python=self.python) if self.conf.test_case != 'fault_management': time.sleep(10) @@ -132,27 +168,44 @@ class ApexInstaller(BaseInstaller): def restore_apply_patches(self): self.log.info('restore apply patches start......') - restart_cmd = 'sudo systemctl restart' \ - ' openstack-ceilometer-notification.service' - restore_scripts = [self.cm_restore_script] + if self.use_containers: + restart_cmd = (self._set_docker_restart_cmd( + "ceilometer-notification")) + restore_scripts.append(self.ac_restore_script) + else: + restart_cmd = 'sudo systemctl restart' \ + ' openstack-ceilometer-notification.service' + if self.conf.test_case != 'fault_management': - restart_cmd += ' openstack-nova-scheduler.service' + if self.use_containers: + restart_cmd += self._set_docker_restart_cmd("nova-scheduler") + else: + restart_cmd += ' openstack-nova-scheduler.service' + restore_scripts.append(self.nc_restore_compute_script) if self.conf.inspector.type == Inspector.CONGRESS: - restart_cmd += ' openstack-congress-server.service' + if self.use_containers: + restart_cmd += self._set_docker_restart_cmd("congress-server") + else: + restart_cmd += ' openstack-congress-server.service' restore_scripts.append(self.cg_restore_script) for client in self.controller_clients: self._run_apply_patches(client, restart_cmd, - restore_scripts) + restore_scripts, + python=self.python) if self.conf.test_case != 'fault_management': - restart_cmd = 'sudo systemctl restart' \ - ' openstack-nova-compute.service' + if self.use_containers: + restart_cmd = self._set_docker_restart_cmd("nova-compute") + else: + restart_cmd = 'sudo systemctl restart' \ + ' openstack-nova-compute.service' for client in self.compute_clients: self._run_apply_patches(client, restart_cmd, - [self.nc_restore_compute_script]) + [self.nc_restore_compute_script], + python=self.python) diff --git a/doctor_tests/installer/base.py b/doctor_tests/installer/base.py index 953b36d9..30435931 100644 --- a/doctor_tests/installer/base.py +++ b/doctor_tests/installer/base.py @@ -26,6 +26,7 @@ class BaseInstaller(object): self.conf = conf self.log = log self.servers = list() + self.use_containers = False @abc.abstractproperty def node_user_name(self): @@ -118,7 +119,20 @@ class BaseInstaller(object): % (output, command, self.conf.installer.type)) return output - def _run_apply_patches(self, client, restart_cmd, script_names): + def _check_cmd_remote(self, client, command): + self.log.info('Check command=%s return in %s installer......' + % (command, self.conf.installer.type)) + + ret, output = client.ssh(command, raise_enabled=False) + self.log.info('return %s' % ret) + if ret == 0: + ret = True + else: + ret = False + return ret + + def _run_apply_patches(self, client, restart_cmd, script_names, + python='python3'): installer_dir = os.path.dirname(os.path.realpath(__file__)) if isinstance(script_names, list): @@ -126,10 +140,10 @@ class BaseInstaller(object): script_abs_path = '{0}/{1}/{2}'.format(installer_dir, 'common', script_name) client.scp(script_abs_path, script_name) - cmd = 'sudo python3 %s' % script_name + cmd = 'sudo %s %s' % (python, script_name) ret, output = client.ssh(cmd) if ret: - raise Exception('Do the command in controller' + raise Exception('Do the command in remote' ' node failed, ret=%s, cmd=%s, output=%s' % (ret, cmd, output)) client.ssh(restart_cmd) diff --git a/doctor_tests/installer/common/restart_aodh.py b/doctor_tests/installer/common/restart_aodh.py new file mode 100644 index 00000000..4473bdca --- /dev/null +++ b/doctor_tests/installer/common/restart_aodh.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2018 Nokia Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import socket +import subprocess + + +def restart_aodh_event_alarm(): + # Restart aodh-evaluator docker with localhost as controller host ip + # This makes our alarm sending look the same as without container + + orig_docker_id = subprocess.check_output("docker ps | grep aodh-evaluator " + "| awk '{print $1}'", shell=True) + get_docker_startup = ( + 'docker inspect --format=\'{{range .Config.Env}} -e "{{.}}" {{end}} ' + '{{range .Mounts}} -v {{.Source}}:{{.Destination}}{{if .Mode}}:' + '{{.Mode}}{{end}}{{end}} -ti {{.Config.Image}}\'' + ) + docker_start = subprocess.check_output("%s %s" % (get_docker_startup, + orig_docker_id), shell=True) + with open("orig_docker_id", 'w') as oid: + oid.write(orig_docker_id) + oid.close() + subprocess.check_output("docker stop %s" % orig_docker_id, shell=True) + ip = socket.gethostbyname(socket.gethostname()) + + ae_start = '-d --add-host="localhost:%s" %s' % (ip, docker_start) + subprocess.check_output("docker run %s" % ae_start, shell=True) + new_docker_id = subprocess.check_output("docker ps | grep aodh-evaluator " + " | awk '{print $1}'", shell=True) + if orig_docker_id == new_docker_id: + raise Exception("Docker ids matching!") + with open("new_docker_id", 'w') as nid: + nid.write(new_docker_id) + nid.close() + +restart_aodh_event_alarm() diff --git a/doctor_tests/installer/common/restore_aodh.py b/doctor_tests/installer/common/restore_aodh.py new file mode 100644 index 00000000..b55eae8d --- /dev/null +++ b/doctor_tests/installer/common/restore_aodh.py @@ -0,0 +1,32 @@ +############################################################################## +# Copyright (c) 2018 Nokia Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import subprocess + + +def restore_aodh_event_alarm(): + # Remove modified docker and restore original + orig = "orig_docker_id" + new = "new_docker_id" + if os.path.isfile(orig): + with open("orig_docker_id", 'r') as oid: + orig_docker_id = oid.read() + oid.close() + if os.path.isfile(new): + with open("new_docker_id", 'r') as nid: + new_docker_id = nid.read() + nid.close() + subprocess.check_output("docker stop %s" % new_docker_id, + shell=True) + subprocess.check_output("docker rm %s" % new_docker_id, shell=True) + os.remove(new) + subprocess.check_output("docker start %s" % orig_docker_id, shell=True) + os.remove(orig) + +restore_aodh_event_alarm() diff --git a/doctor_tests/installer/common/restore_compute_config.py b/doctor_tests/installer/common/restore_compute_config.py index 0971d12b..0e9939fd 100644 --- a/doctor_tests/installer/common/restore_compute_config.py +++ b/doctor_tests/installer/common/restore_compute_config.py @@ -11,8 +11,11 @@ import shutil def restore_cpu_allocation_ratio(): - nova_file = '/etc/nova/nova.conf' - nova_file_bak = '/etc/nova/nova.bak' + nova_base = "/var/lib/config-data/puppet-generated/nova" + if not os.path.isdir(nova_base): + nova_base = "" + nova_file = nova_base + '/etc/nova/nova.conf' + nova_file_bak = nova_base + '/etc/nova/nova.bak' if not os.path.isfile(nova_file_bak): print('Bak_file:%s does not exist.' % nova_file_bak) diff --git a/doctor_tests/installer/common/restore_config.py b/doctor_tests/installer/common/restore_config.py index c1f919c1..5cb83b27 100644 --- a/doctor_tests/installer/common/restore_config.py +++ b/doctor_tests/installer/common/restore_config.py @@ -9,11 +9,15 @@ import os import shutil -ep_file = '/etc/ceilometer/event_pipeline.yaml' -ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak' + +cbase = "/var/lib/config-data/puppet-generated/ceilometer" +if not os.path.isdir(cbase): + cbase = "" def restore_ep_config(): + ep_file = cbase + '/etc/ceilometer/event_pipeline.yaml' + ep_file_bak = cbase + '/etc/ceilometer/event_pipeline.yaml.bak' if not os.path.isfile(ep_file_bak): print('Bak_file:%s does not exist.' % ep_file_bak) @@ -25,31 +29,20 @@ def restore_ep_config(): def restore_ed_config(): - - ed_file = '/etc/ceilometer/event_definitions.yaml' - ed_file_bak = '/etc/ceilometer/event_definitions.bak' + ed_file = cbase + '/etc/ceilometer/event_definitions.yaml' + ed_file_bak = cbase + '/etc/ceilometer/event_definitions.bak' if not os.path.isfile(ed_file_bak): print("Bak_file doesn't exist: %s." % ed_file_bak) else: print('restore: %s' % ed_file) - shutil.copyfile(ed_file_bak, ed_file) + if os.stat(ed_file_bak).st_size == 0: + print('Bak_file empty, so removing also: %s' % ed_file) + os.remove(ed_file) + else: + shutil.copyfile(ed_file_bak, ed_file) os.remove(ed_file_bak) return - -def restore_cpu_allocation_ratio(): - nova_file = '/etc/nova/nova.conf' - nova_file_bak = '/etc/nova/nova.bak' - - if not os.path.isfile(nova_file_bak): - print('Bak_file:%s does not exist.' % nova_file_bak) - else: - print('restore: %s' % nova_file) - shutil.copyfile(nova_file_bak, nova_file) - os.remove(nova_file_bak) - return - restore_ep_config() restore_ed_config() -restore_cpu_allocation_ratio() diff --git a/doctor_tests/installer/common/restore_congress.py b/doctor_tests/installer/common/restore_congress.py index b5efb1ef..576f1b16 100644 --- a/doctor_tests/installer/common/restore_congress.py +++ b/doctor_tests/installer/common/restore_congress.py @@ -11,8 +11,11 @@ import shutil def restore_drivers_config(): - co_conf = "/etc/congress/congress.conf" - co_conf_bak = "/etc/congress/congress.conf.bak" + co_base = "/var/lib/config-data/puppet-generated/congress" + if not os.path.isdir(co_base): + co_base = "" + co_conf = co_base + "/etc/congress/congress.conf" + co_conf_bak = co_base + "/etc/congress/congress.conf.bak" if not os.path.isfile(co_conf_bak): print('Bak_file:%s does not exist.' % co_conf_bak) diff --git a/doctor_tests/installer/common/set_compute_config.py b/doctor_tests/installer/common/set_compute_config.py index 07db1e16..86266085 100644 --- a/doctor_tests/installer/common/set_compute_config.py +++ b/doctor_tests/installer/common/set_compute_config.py @@ -10,9 +10,26 @@ import os import shutil +def make_initial_config(service, dest): + for mk in ["", "/etc", "/%s" % service]: + dest += mk + os.mkdir(dest) + src = "/etc/%s/%s.conf" % (service, service) + dest += "/%s.conf" % service + shutil.copyfile(src, dest) + + def set_cpu_allocation_ratio(): - nova_file = '/etc/nova/nova.conf' - nova_file_bak = '/etc/nova/nova.bak' + docker_conf_base_dir = "/var/lib/config-data/puppet-generated" + if not os.path.isdir(docker_conf_base_dir): + nova_base = "" + else: + nova_base = "%s/nova" % docker_conf_base_dir + if not os.path.isdir(nova_base): + # nova.conf to be used might not exist + make_initial_config("nova", nova_base) + nova_file = nova_base + '/etc/nova/nova.conf' + nova_file_bak = nova_base + '/etc/nova/nova.bak' if not os.path.isfile(nova_file): raise Exception("File doesn't exist: %s." % nova_file) diff --git a/doctor_tests/installer/common/set_config.py b/doctor_tests/installer/common/set_config.py index 42465247..3dc6cd9a 100644 --- a/doctor_tests/installer/common/set_config.py +++ b/doctor_tests/installer/common/set_config.py @@ -10,12 +10,16 @@ import os import shutil import yaml -ep_file = '/etc/ceilometer/event_pipeline.yaml' -ep_file_bak = '/etc/ceilometer/event_pipeline.yaml.bak' -event_notifier_topic = 'notifier://?topic=alarm.all' + +cbase = "/var/lib/config-data/puppet-generated/ceilometer" +if not os.path.isdir(cbase): + cbase = "" def set_notifier_topic(): + ep_file = cbase + '/etc/ceilometer/event_pipeline.yaml' + ep_file_bak = cbase + '/etc/ceilometer/event_pipeline.yaml.bak' + event_notifier_topic = 'notifier://?topic=alarm.all' config_modified = False if not os.path.isfile(ep_file): @@ -42,24 +46,77 @@ def set_notifier_topic(): file.write(yaml.safe_dump(config)) -def set_maintenance_event_definitions(): - ed_file = '/etc/ceilometer/event_definitions.yaml' - ed_file_bak = '/etc/ceilometer/event_definitions.bak' +def set_event_definitions(): + ed_file = cbase + '/etc/ceilometer/event_definitions.yaml' + ed_file_bak = cbase + '/etc/ceilometer/event_definitions.bak' + orig_ed_file_exist = True + modify_config = False if not os.path.isfile(ed_file): - raise Exception("File doesn't exist: %s." % ed_file) - - with open(ed_file, 'r') as file: - config = yaml.safe_load(file) + # Deployment did not modify file, so it did not exist + src_file = '/etc/ceilometer/event_definitions.yaml' + if not os.path.isfile(src_file): + config = [] + orig_ed_file_exist = False + else: + shutil.copyfile('/etc/ceilometer/event_definitions.yaml', ed_file) + if orig_ed_file_exist: + with open(ed_file, 'r') as file: + config = yaml.safe_load(file) et_list = [et['event_type'] for et in config] + if 'compute.instance.update' in et_list: + print('NOTE: compute.instance.update allready configured') + else: + print('NOTE: add compute.instance.update to event_definitions.yaml') + modify_config = True + instance_update = { + 'event_type': 'compute.instance.update', + 'traits': { + 'deleted_at': {'fields': 'payload.deleted_at', + 'type': 'datetime'}, + 'disk_gb': {'fields': 'payload.disk_gb', + 'type': 'int'}, + 'display_name': {'fields': 'payload.display_name'}, + 'ephemeral_gb': {'fields': 'payload.ephemeral_gb', + 'type': 'int'}, + 'host': {'fields': 'publisher_id.`split(., 1, 1)`'}, + 'instance_id': {'fields': 'payload.instance_id'}, + 'instance_type': {'fields': 'payload.instance_type'}, + 'instance_type_id': {'fields': 'payload.instance_type_id', + 'type': 'int'}, + 'launched_at': {'fields': 'payload.launched_at', + 'type': 'datetime'}, + 'memory_mb': {'fields': 'payload.memory_mb', + 'type': 'int'}, + 'old_state': {'fields': 'payload.old_state'}, + 'os_architecture': { + 'fields': + "payload.image_meta.'org.openstack__1__architecture'"}, + 'os_distro': { + 'fields': + "payload.image_meta.'org.openstack__1__os_distro'"}, + 'os_version': { + 'fields': + "payload.image_meta.'org.openstack__1__os_version'"}, + 'resource_id': {'fields': 'payload.instance_id'}, + 'root_gb': {'fields': 'payload.root_gb', + 'type': 'int'}, + 'service': {'fields': 'publisher_id.`split(., 0, -1)`'}, + 'state': {'fields': 'payload.state'}, + 'tenant_id': {'fields': 'payload.tenant_id'}, + 'user_id': {'fields': 'payload.user_id'}, + 'vcpus': {'fields': 'payload.vcpus', 'type': 'int'} + } + } + config.append(instance_update) + if 'maintenance.scheduled' in et_list: - add_mscheduled = False print('NOTE: maintenance.scheduled allready configured') else: print('NOTE: add maintenance.scheduled to event_definitions.yaml') - add_mscheduled = True + modify_config = True mscheduled = { 'event_type': 'maintenance.scheduled', 'traits': { @@ -72,16 +129,15 @@ def set_maintenance_event_definitions(): 'session_id': {'fields': 'payload.session_id'}, 'project_id': {'fields': 'payload.project_id'}, 'metadata': {'fields': 'payload.metadata'} + } } - } config.append(mscheduled) if 'maintenance.host' in et_list: - add_mhost = False print('NOTE: maintenance.host allready configured') else: print('NOTE: add maintenance.host to event_definitions.yaml') - add_mhost = True + modify_config = True mhost = { 'event_type': 'maintenance.host', 'traits': { @@ -89,51 +145,18 @@ def set_maintenance_event_definitions(): 'project_id': {'fields': 'payload.project_id'}, 'state': {'fields': 'payload.state'}, 'session_id': {'fields': 'payload.session_id'} + } } - } config.append(mhost) - if add_mscheduled or add_mhost: - shutil.copyfile(ed_file, ed_file_bak) + if modify_config: + if orig_ed_file_exist: + shutil.copyfile(ed_file, ed_file_bak) + else: + with open(ed_file_bak, 'w+') as file: + file.close() with open(ed_file, 'w+') as file: file.write(yaml.safe_dump(config)) - -def set_cpu_allocation_ratio(): - nova_file = '/etc/nova/nova.conf' - nova_file_bak = '/etc/nova/nova.bak' - - if not os.path.isfile(nova_file): - raise Exception("File doesn't exist: %s." % nova_file) - # TODO (tojuvone): Unfortunately ConfigParser did not produce working conf - fcheck = open(nova_file) - found_list = ([ca for ca in fcheck.readlines() if "cpu_allocation_ratio" - in ca]) - fcheck.close() - if found_list and len(found_list): - change = False - found = False - for car in found_list: - if car.startswith('#'): - continue - if car.startswith('cpu_allocation_ratio'): - found = True - if "1.0" not in car.split('=')[1]: - change = True - if not found or change: - # need to add or change - shutil.copyfile(nova_file, nova_file_bak) - fin = open(nova_file_bak) - fout = open(nova_file, "wt") - for line in fin: - if change and line.startswith("cpu_allocation_ratio"): - line = "cpu_allocation_ratio=1.0" - if not found and line.startswith("[DEFAULT]"): - line += "cpu_allocation_ratio=1.0\n" - fout.write(line) - fin.close() - fout.close() - set_notifier_topic() -set_maintenance_event_definitions() -set_cpu_allocation_ratio() +set_event_definitions() diff --git a/doctor_tests/installer/common/set_congress.py b/doctor_tests/installer/common/set_congress.py index d8438701..d2754838 100644 --- a/doctor_tests/installer/common/set_congress.py +++ b/doctor_tests/installer/common/set_congress.py @@ -7,12 +7,16 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import configparser +import os import shutil def set_drivers_config(): - co_conf = "/etc/congress/congress.conf" - co_conf_bak = "/etc/congress/congress.conf.bak" + co_base = "/var/lib/config-data/puppet-generated/congress" + if not os.path.isdir(co_base): + co_base = "" + co_conf = co_base + "/etc/congress/congress.conf" + co_conf_bak = co_base + "/etc/congress/congress.conf.bak" doctor_driver = "congress.datasources.doctor_driver.DoctorDriver" config_modified = False diff --git a/doctor_tests/installer/common/vitrage.py b/doctor_tests/installer/common/vitrage.py index 30a73f5d..801adff5 100644 --- a/doctor_tests/installer/common/vitrage.py +++ b/doctor_tests/installer/common/vitrage.py @@ -9,8 +9,11 @@ import os +vi_base = "/var/lib/config-data/puppet-generated/vitrage" +if not os.path.isdir(vi_base): + vi_base = "" vitrage_template_file = \ - '/etc/vitrage/templates/vitrage_host_down_scenarios.yaml' + vi_base + '/etc/vitrage/templates/vitrage_host_down_scenarios.yaml' template = """ metadata: diff --git a/doctor_tests/scenario/maintenance.py b/doctor_tests/scenario/maintenance.py index 54244d79..9fcd4128 100644 --- a/doctor_tests/scenario/maintenance.py +++ b/doctor_tests/scenario/maintenance.py @@ -168,9 +168,9 @@ class Maintenance(object): return ret.json()['state'] def wait_maintenance_complete(self, session_id): - retries = 60 + retries = 66 state = None - time.sleep(600) + time.sleep(540) while state != 'MAINTENANCE_COMPLETE' and retries > 0: time.sleep(10) state = self.get_maintenance_state(session_id) diff --git a/doctor_tests/stack.py b/doctor_tests/stack.py index 688c2050..ee586fa8 100644 --- a/doctor_tests/stack.py +++ b/doctor_tests/stack.py @@ -44,7 +44,7 @@ class Stack(object): action_failed = '%s_FAILED' % action status = action_in_progress - stack_retries = 150 + stack_retries = 160 while status == action_in_progress and stack_retries > 0: time.sleep(2) try: @@ -88,7 +88,19 @@ class Stack(object): template=template, parameters=parameters) self.stack_id = stack['stack']['id'] - self.wait_stack_create() + try: + self.wait_stack_create() + except Exception: + # It might not always work at first + self.log.info('retry creating maintenance stack.......') + self.delete() + time.sleep(3) + stack = self.heat.stacks.create(stack_name=self.stack_name, + files=files, + template=template, + parameters=parameters) + self.stack_id = stack['stack']['id'] + self.wait_stack_create() def update(self, stack_name, stack_id, template, parameters={}, files={}): self.heat.stacks.update(stack_name=stack_name, |