diff options
Diffstat (limited to 'doctor_tests/scenario')
| -rw-r--r-- | doctor_tests/scenario/fault_management.py | 2 | ||||
| -rw-r--r-- | doctor_tests/scenario/maintenance.py | 192 |
2 files changed, 193 insertions, 1 deletions
diff --git a/doctor_tests/scenario/fault_management.py b/doctor_tests/scenario/fault_management.py index b1fe8099..f8f53e8e 100644 --- a/doctor_tests/scenario/fault_management.py +++ b/doctor_tests/scenario/fault_management.py @@ -32,7 +32,7 @@ dev=$(sudo ip a | awk '/ {compute_ip}\//{{print $NF}}') sleep 1 sudo ip link set $dev down echo "doctor set link down at" $(date "+%s.%N") -sleep 10 +sleep 30 sudo ip link set $dev up sleep 1 """ diff --git a/doctor_tests/scenario/maintenance.py b/doctor_tests/scenario/maintenance.py new file mode 100644 index 00000000..54244d79 --- /dev/null +++ b/doctor_tests/scenario/maintenance.py @@ -0,0 +1,192 @@ +############################################################################## +# Copyright (c) 2018 Nokia Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import datetime +import json +import requests +import time + +from doctor_tests.admin_tool import get_admin_tool +from doctor_tests.app_manager import get_app_manager +from doctor_tests.common.utils import get_doctor_test_root_dir +from doctor_tests.identity_auth import get_identity_auth +from doctor_tests.identity_auth import get_session +from doctor_tests.inspector import get_inspector +from doctor_tests.os_clients import keystone_client +from doctor_tests.os_clients import neutron_client +from doctor_tests.os_clients import nova_client +from doctor_tests.stack import Stack + + +class Maintenance(object): + + def __init__(self, trasport_url, conf, log): + self.conf = conf + self.log = log + self.keystone = keystone_client( + self.conf.keystone_version, get_session()) + self.nova = nova_client(conf.nova_version, get_session()) + auth = get_identity_auth(project=self.conf.doctor_project) + self.neutron = neutron_client(get_session(auth=auth)) + self.stack = Stack(self.conf, self.log) + self.admin_tool = get_admin_tool(trasport_url, self.conf, self.log) + self.app_manager = get_app_manager(self.stack, self.conf, self.log) + self.inspector = get_inspector(self.conf, self.log) + + def get_external_network(self): + ext_net = None + networks = self.neutron.list_networks()['networks'] + for network in networks: + if network['router:external']: + ext_net = network['name'] + break + if ext_net is None: + raise Exception("external network not defined") + return ext_net + + def setup_maintenance(self, user): + # each hypervisor needs to have same amount of vcpus and they + # need to be free before test + hvisors = self.nova.hypervisors.list(detailed=True) + prev_vcpus = 0 + prev_hostname = '' + self.log.info('checking hypervisors.......') + for hvisor in hvisors: + vcpus = hvisor.__getattr__('vcpus') + vcpus_used = hvisor.__getattr__('vcpus_used') + hostname = hvisor.__getattr__('hypervisor_hostname') + if vcpus < 2: + raise Exception('not enough vcpus (%d) on %s' % + (vcpus, hostname)) + if vcpus_used > 0: + raise Exception('%d vcpus used on %s' + % (vcpus_used, hostname)) + if prev_vcpus != 0 and prev_vcpus != vcpus: + raise Exception('%d vcpus on %s does not match to' + '%d on %s' + % (vcpus, hostname, + prev_vcpus, prev_hostname)) + prev_vcpus = vcpus + prev_hostname = hostname + + # maintenance flavor made so that 2 instances take whole node + flavor_vcpus = int(vcpus / 2) + compute_nodes = len(hvisors) + amount_actstdby_instances = 2 + amount_noredundancy_instances = 2 * compute_nodes - 2 + self.log.info('testing %d computes with %d vcpus each' + % (compute_nodes, vcpus)) + self.log.info('testing %d actstdby and %d noredundancy instances' + % (amount_actstdby_instances, + amount_noredundancy_instances)) + max_instances = (amount_actstdby_instances + + amount_noredundancy_instances) + max_cores = compute_nodes * vcpus + + user.update_quota(max_instances, max_cores) + + test_dir = get_doctor_test_root_dir() + template_file = '{0}/{1}'.format(test_dir, 'maintenance_hot_tpl.yaml') + files, template = self.stack.get_hot_tpl(template_file) + + ext_net = self.get_external_network() + + parameters = {'ext_net': ext_net, + 'flavor_vcpus': flavor_vcpus, + 'maint_image': self.conf.image_name, + 'nonha_intances': amount_noredundancy_instances, + 'ha_intances': amount_actstdby_instances} + + self.log.info('creating maintenance stack.......') + self.log.info('parameters: %s' % parameters) + + self.stack.create('doctor_test_maintenance', + template, + parameters=parameters, + files=files) + + self.admin_tool.start() + self.app_manager.start() + self.inspector.start() + + def start_maintenance(self): + self.log.info('start maintenance.......') + hvisors = self.nova.hypervisors.list(detailed=True) + maintenance_hosts = list() + for hvisor in hvisors: + hostname = hvisor.__getattr__('hypervisor_hostname') + maintenance_hosts.append(hostname) + + url = 'http://0.0.0.0:%s/maintenance' % self.conf.admin_tool.port + # let's start maintenance 20sec from now, so projects will have + # time to ACK to it before that + maintenance_at = (datetime.datetime.utcnow() + + datetime.timedelta(seconds=20) + ).strftime('%Y-%m-%d %H:%M:%S') + data = {'hosts': maintenance_hosts, + 'state': 'MAINTENANCE', + 'maintenance_at': maintenance_at, + 'metadata': {'openstack_version': 'Pike'}} + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json'} + + ret = requests.post(url, data=json.dumps(data), headers=headers) + if ret.status_code != 200: + raise Exception(ret.text) + return ret.json()['session_id'] + + def remove_maintenance_session(self, session_id): + self.log.info('remove maintenance session %s.......' % session_id) + + url = 'http://0.0.0.0:%s/maintenance' % self.conf.admin_tool.port + + data = {'state': 'REMOVE_MAINTENANCE_SESSION', + 'session_id': session_id} + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json'} + + ret = requests.post(url, data=json.dumps(data), headers=headers) + if ret.status_code != 200: + raise Exception(ret.text) + + def get_maintenance_state(self, session_id): + url = 'http://0.0.0.0:%s/maintenance' % self.conf.admin_tool.port + data = {'session_id': session_id} + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json'} + ret = requests.get(url, data=json.dumps(data), headers=headers) + if ret.status_code != 200: + raise Exception(ret.text) + return ret.json()['state'] + + def wait_maintenance_complete(self, session_id): + retries = 60 + state = None + time.sleep(600) + while state != 'MAINTENANCE_COMPLETE' and retries > 0: + time.sleep(10) + state = self.get_maintenance_state(session_id) + retries = retries - 1 + if retries == 0 and state != 'MAINTENANCE_COMPLETE': + raise Exception('maintenance %s not completed within 20min, status' + ' %s' % (session_id, state)) + elif state == 'MAINTENANCE_COMPLETE': + self.log.info('maintenance %s %s' % (session_id, state)) + self.remove_maintenance_session(session_id) + elif state == 'MAINTENANCE_FAILED': + raise Exception('maintenance %s failed' % session_id) + + def cleanup_maintenance(self): + self.admin_tool.stop() + self.app_manager.stop() + self.inspector.stop() + self.log.info('stack delete start.......') + self.stack.delete() |