From 19f7ba75850c52e1ae163766b64b6d153e8d7e1b Mon Sep 17 00:00:00 2001 From: dongwenjuan Date: Mon, 14 Aug 2017 16:58:13 +0800 Subject: refactor failure inject JIRA: DOCTOR-116 Change-Id: I14deda4ccb47414cff139a522a9196b68e92500e Signed-off-by: dongwenjuan --- tests/scenario/__init__.py | 8 +++++ tests/scenario/common.py | 40 ++++++++++++++++++++++ tests/scenario/network_failure.py | 71 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 tests/scenario/__init__.py create mode 100644 tests/scenario/common.py create mode 100644 tests/scenario/network_failure.py (limited to 'tests/scenario') diff --git a/tests/scenario/__init__.py b/tests/scenario/__init__.py new file mode 100644 index 00000000..48893ae6 --- /dev/null +++ b/tests/scenario/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/tests/scenario/common.py b/tests/scenario/common.py new file mode 100644 index 00000000..e880e8b2 --- /dev/null +++ b/tests/scenario/common.py @@ -0,0 +1,40 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import re +import sys + + +def match_rep_in_file(regex, full_path): + if not os.path.isfile(full_path): + raise Exception('File(%s) does not exist' % full_path) + + with open(full_path, 'r') as file: + for line in file: + result = re.search(regex, line) + if result: + return result.group(0) + + return None + + +def calculate_notification_time(): + log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log') + + reg = '(?<=doctor monitor detected at )\d+.\d+' + detected = match_rep_in_file(reg, log_file) + if not detected: + raise Exception('Can not find detected time') + + reg = '(?<=doctor consumer notified at )\d+.\d+' + notified = match_rep_in_file(reg, log_file) + if not notified: + raise Exception('Can not find notified time') + + return float(notified) - float(detected) \ No newline at end of file diff --git a/tests/scenario/network_failure.py b/tests/scenario/network_failure.py new file mode 100644 index 00000000..1d9027a2 --- /dev/null +++ b/tests/scenario/network_failure.py @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from identity_auth import get_session +from os_clients import nova_client +from common.utils import SSHClient + +LINK_DOWN_SCRIPT = """ +#!/bin/bash -x +dev=$(sudo ip a | awk '/ {compute_ip}\//{{print $NF}}') +sleep 1 +sudo ip link set $dev down +echo "doctor set link down at" $(date "+%s.%N") +sleep 10 +sudo ip link set $dev up +sleep 1 +""" + + +class NetworkFault(object): + + def __init__(self, conf, installer, log): + self.conf = conf + self.log = log + self.installer = installer + self.nova = nova_client(self.conf.nova_version, get_session()) + self.host = None + self.GetLog = False + + def start(self, host): + self.log.info('fault inject start......') + self._set_link_down(host.ip) + self.host = host + self.log.info('fault inject end......') + + def cleanup(self): + self.log.info('fault inject cleanup......') + self.get_diable_network_log() + + def get_diable_network_log(self): + if not self.GetLog: + self.log.info('Already get the disable_netork.log from down_host......') + return + if self.host is not None: + client = SSHClient(self.host.ip, + self.installer.node_user_name, + key_filename=self.installer.get_ssh_key_from_installer(), + look_for_keys=True, + log=self.log) + client.scp('disable_network.log', './disable_network.log', method='get') + self.log.info('Get the disable_netork.log from down_host(host_name:%s, host_ip:%s)' + % (self.host.name, self.host.ip)) + self.GetLog = True + + def _set_link_down(self, compute_ip): + file_name = './disable_network.sh' + with open(file_name, 'w') as file: + file.write(LINK_DOWN_SCRIPT.format(compute_ip=compute_ip)) + client = SSHClient(compute_ip, + self.installer.node_user_name, + key_filename=self.installer.get_ssh_key_from_installer(), + look_for_keys=True, + log=self.log) + client.scp('./disable_network.sh', 'disable_network.sh') + command = 'bash disable_network.sh > disable_network.log 2>&1 &' + client.ssh(command) -- cgit 1.2.3-korg