diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/common/utils.py | 14 | ||||
| -rw-r--r-- | tests/config.py | 4 | ||||
| -rw-r--r-- | tests/main.py | 40 | ||||
| -rw-r--r-- | tests/profiler_poc.py (renamed from tests/profiler-poc.py) | 23 | ||||
| -rwxr-xr-x | tests/run.sh | 2 | ||||
| -rw-r--r-- | tests/scenario/common.py | 30 | ||||
| -rw-r--r-- | tests/scenario/network_failure.py | 6 |
7 files changed, 88 insertions, 31 deletions
diff --git a/tests/common/utils.py b/tests/common/utils.py index 38fd97d8..2e823acb 100644 --- a/tests/common/utils.py +++ b/tests/common/utils.py @@ -9,6 +9,7 @@ import json import os import paramiko +import re def load_json_file(full_path): @@ -33,6 +34,19 @@ def write_json_file(full_path, data): file.write(json.dumps(data)) +def match_rep_in_file(regex, full_path): + if not os.path.isfile(full_path): + raise Exception('File(%s) does not exist' % full_path) + + with open(full_path, 'r') as file: + for line in file: + result = re.search(regex, line) + if result: + return result + + return None + + class SSHClient(object): def __init__(self, ip, username, password=None, pkey=None, key_filename=None, log=None, look_for_keys=False, diff --git a/tests/config.py b/tests/config.py index f33ab5d2..c71d5ad7 100644 --- a/tests/config.py +++ b/tests/config.py @@ -19,6 +19,7 @@ import network import inspector
import monitor
import os_clients
+import profiler_poc
import user
@@ -34,7 +35,8 @@ def list_opts(): user.OPTS,
network.OPTS,
instance.OPTS,
- alarm.OPTS))
+ alarm.OPTS,
+ profiler_poc.OPTS))
]
diff --git a/tests/main.py b/tests/main.py index 7e7c3bc2..df7e95f3 100644 --- a/tests/main.py +++ b/tests/main.py @@ -14,6 +14,7 @@ import time from alarm import Alarm from common.constants import Host +from common.utils import match_rep_in_file import config from consumer import get_consumer from identity_auth import get_identity_auth @@ -26,6 +27,7 @@ import logger as doctor_log from network import Network from monitor import get_monitor from os_clients import nova_client +from profiler_poc import main as profiler_main from scenario.common import calculate_notification_time from scenario.network_failure import NetworkFault from user import User @@ -110,6 +112,11 @@ class DoctorTest(object): else: LOG.error('doctor test failed, notification_time=%s' % notification_time) sys.exit(1) + + if self.conf.profiler_type: + LOG.info('doctor test begin to run profile.......') + self.collect_logs() + self.run_profiler() except Exception as e: LOG.error('doctor test failed, Exception=%s' % e) sys.exit(1) @@ -145,6 +152,39 @@ class DoctorTest(object): time.sleep(2) self.check_host_status(self.down_host.name, 'up') + def collect_logs(self): + self.fault.get_disable_network_log() + + def run_profiler(self): + + log_file = '{0}/{1}'.format(sys.path[0], 'disable_network.log') + reg = '(?<=doctor set link down at )\d+.\d+' + linkdown = float(match_rep_in_file(reg, log_file).group(0)) + + log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log') + reg = '(.* doctor mark vm.* error at )(\d+.\d+)' + vmdown = float(match_rep_in_file(reg, log_file).group(2)) + + reg = '(?<=doctor mark host.* down at )\d+.\d+' + hostdown = float(match_rep_in_file(reg, log_file).group(2)) + + reg = '(?<=doctor monitor detected at )\d+.\d+' + detected = float(match_rep_in_file(reg, log_file).group(0)) + + reg = '(?<=doctor consumer notified at )\d+.\d+' + notified = float(match_rep_in_file(reg, log_file).group(0)) + + # TODO(yujunz) check the actual delay to verify time sync status + # expected ~1s delay from $trigger to $linkdown + relative_start = linkdown + os.environ['DOCTOR_PROFILER_T00'] = str(int((linkdown - relative_start)*1000)) + os.environ['DOCTOR_PROFILER_T01'] = str(int((detected - relative_start) * 1000)) + os.environ['DOCTOR_PROFILER_T03'] = str(int((vmdown - relative_start) * 1000)) + os.environ['DOCTOR_PROFILER_T04'] = str(int((hostdown - relative_start) * 1000)) + os.environ['DOCTOR_PROFILER_T09'] = str(int((notified - relative_start) * 1000)) + + profiler_main(log=LOG) + def cleanup(self): self.unset_forced_down_hosts() self.inspector.stop() diff --git a/tests/profiler-poc.py b/tests/profiler_poc.py index 408cb09a..ea36eaeb 100644 --- a/tests/profiler-poc.py +++ b/tests/profiler_poc.py @@ -21,6 +21,16 @@ See also: https://goo.gl/98Osig import json import os +from oslo_config import cfg + + +OPTS = [ + cfg.StrOpt('profiler_type', + default=os.environ.get('PROFILER_TYPE', 'poc'), + help='the type of installer'), +] + + OUTPUT = 'doctor_profiling_output' PREFIX = 'DOCTOR_PROFILER' TOTAL_CHECK_POINTS = 10 @@ -46,7 +56,7 @@ link down:{T00}| | | | | | | | | """ -def main(): +def main(log=None): check_points = ["T{:02d}".format(i) for i in range(TOTAL_CHECK_POINTS)] module_map = {"M{:02d}".format(i): (MODULE_CHECK_POINTS[i], MODULE_CHECK_POINTS[i + 1]) @@ -59,7 +69,7 @@ def main(): def format_tag(tag): return TAG_FORMAT.format(tag or '?') - tags = {cp: format_tag(ms) for cp, ms in elapsed_ms.iteritems()} + tags = {cp: format_tag(ms) for cp, ms in elapsed_ms.items()} def time_cost(cp): if elapsed_ms[cp[0]] and elapsed_ms[cp[1]]: @@ -69,10 +79,10 @@ def main(): # module time cost tags modules_cost_ms = {module: time_cost(cp) - for module, cp in module_map.iteritems()} + for module, cp in module_map.items()} tags.update({module: format_tag(cost) - for module, cost in modules_cost_ms.iteritems()}) + for module, cost in modules_cost_ms.items()}) tags.update({'total': time_cost((check_points[0], check_points[-1]))}) @@ -81,7 +91,10 @@ def main(): logfile = open('{}.json'.format(OUTPUT), 'w') logfile.write(json.dumps(tags)) - print profile + print(profile) + if log: + log.info('%s' % profile) + if __name__ == '__main__': main() diff --git a/tests/run.sh b/tests/run.sh index dceb0614..e1875e09 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -421,7 +421,7 @@ run_profiler() { export DOCTOR_PROFILER_T09=$(python -c \ "print(int(($notified-$relative_start)*1000))") - python profiler-poc.py > doctor_profiler.log 2>&1 + python profiler_poc.py > doctor_profiler.log 2>&1 fi } diff --git a/tests/scenario/common.py b/tests/scenario/common.py index e880e8b2..a33c50ff 100644 --- a/tests/scenario/common.py +++ b/tests/scenario/common.py @@ -6,35 +6,23 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -import os -import re import sys - - -def match_rep_in_file(regex, full_path): - if not os.path.isfile(full_path): - raise Exception('File(%s) does not exist' % full_path) - - with open(full_path, 'r') as file: - for line in file: - result = re.search(regex, line) - if result: - return result.group(0) - - return None +from common.utils import match_rep_in_file def calculate_notification_time(): log_file = '{0}/{1}'.format(sys.path[0], 'doctor.log') reg = '(?<=doctor monitor detected at )\d+.\d+' - detected = match_rep_in_file(reg, log_file) - if not detected: - raise Exception('Can not find detected time') + result = match_rep_in_file(reg, log_file) + if not result: + raise Exception('Can not match detected time') + detected = result.group(0) reg = '(?<=doctor consumer notified at )\d+.\d+' - notified = match_rep_in_file(reg, log_file) - if not notified: - raise Exception('Can not find notified time') + result = match_rep_in_file(reg, log_file) + if not result: + raise Exception('Can not match notified time') + notified = result.group(0) return float(notified) - float(detected)
\ No newline at end of file diff --git a/tests/scenario/network_failure.py b/tests/scenario/network_failure.py index 1d9027a2..e9a239db 100644 --- a/tests/scenario/network_failure.py +++ b/tests/scenario/network_failure.py @@ -40,10 +40,10 @@ class NetworkFault(object): def cleanup(self): self.log.info('fault inject cleanup......') - self.get_diable_network_log() + self.get_disable_network_log() - def get_diable_network_log(self): - if not self.GetLog: + def get_disable_network_log(self): + if self.GetLog: self.log.info('Already get the disable_netork.log from down_host......') return if self.host is not None: |