diff options
author | wym_libra <yimin.wang@huawei.com> | 2015-12-07 16:02:18 +0800 |
---|---|---|
committer | qi liang <liangqi1@huawei.com> | 2016-01-10 03:16:07 +0000 |
commit | 2733defda816a84d2b9c2e361a5970b9de4923f4 (patch) | |
tree | ab4a5507c4100cec6485ba9d587aca8728e8d1c7 /yardstick/benchmark/scenarios/availability/monitor | |
parent | d93f6ca5bf0d4f22ec0fd90eac1bf8b9c09db820 (diff) |
Rewrite the HA test case (2)
idea: refact the Monitor class in old file "monitor.py" with the base
class and sub-class.
detail:
1) the BaseMonitor is the base class of other monitor
2) each monitor run in independent process
3) there are two monitor("openstack-cmd" and "process") for the first test case
4) MonitorMgr class used to manager monitor process
JIRA: YARDSTICK-149
Change-Id: I2eede94481f740812212e6cb673d175b5f543c15
Signed-off-by: wym_libra <yimin.wang@huawei.com>
Diffstat (limited to 'yardstick/benchmark/scenarios/availability/monitor')
6 files changed, 332 insertions, 0 deletions
diff --git a/yardstick/benchmark/scenarios/availability/monitor/__init__.py b/yardstick/benchmark/scenarios/availability/monitor/__init__.py new file mode 100755 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/__init__.py diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py new file mode 100644 index 000000000..25990ac8c --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py @@ -0,0 +1,140 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import pkg_resources +import logging +import multiprocessing +import time +import os +import yardstick.common.utils as utils + +LOG = logging.getLogger(__name__) + +monitor_conf_path = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.availability.monitor", + "monitor_conf.yaml") + + +class BaseMonitor(multiprocessing.Process): + """docstring for BaseMonitor""" + + def __init__(self, config, context): + multiprocessing.Process.__init__(self) + self._config = config + self._context = context + self._queue = multiprocessing.Queue() + self._event = multiprocessing.Event() + self.setup_done = False + + @staticmethod + def get_monitor_cls(monitor_type): + '''return monitor class of specified type''' + + for monitor in utils.itersubclasses(BaseMonitor): + if monitor_type == monitor.__monitor_type__: + return monitor + raise RuntimeError("No such monitor_type %s" % monitor_type) + + def get_script_fullpath(self, path): + base_path = os.path.dirname(monitor_conf_path) + return os.path.join(base_path, path) + + def run(self): + LOG.debug("config:%s context:%s" % (self._config, self._context)) + + self.setup() + monitor_time = self._config.get("monitor_time", 0) + + total_time = 0 + outage_time = 0 + total_count = 0 + outage_count = 0 + first_outage = 0 + last_outage = 0 + + begin_time = time.time() + while True: + total_count = total_count + 1 + + one_check_begin_time = time.time() + exit_status = self.monitor_func() + one_check_end_time = time.time() + + if exit_status is False: + outage_count = outage_count + 1 + + outage_time = outage_time + ( + one_check_end_time - one_check_begin_time) + + if not first_outage: + first_outage = one_check_begin_time + + last_outage = one_check_end_time + + if self._event.is_set(): + LOG.debug("the monitor process stop") + break + + if one_check_end_time - begin_time > monitor_time: + LOG.debug("the monitor max_time finished and exit!") + break + + end_time = time.time() + total_time = end_time - begin_time + + self._queue.put({"total_time": total_time, + "outage_time": last_outage-first_outage, + "total_count": total_count, + "outage_count": outage_count}) + + def start_monitor(self): + self.start() + + def wait_monitor(self): + self.join() + self._result = self._queue.get() + LOG.debug("the monitor result:%s" % self._result) + + def setup(self): # pragma: no cover + pass + + def monitor_func(self): # pragma: no cover + pass + + def verify_SLA(self): + pass + + +class MonitorMgr(object): + """docstring for MonitorMgr""" + def __init__(self): + self._monitor_list = [] + + def init_monitors(self, monitor_cfgs, context): + LOG.debug("monitorMgr config: %s" % monitor_cfgs) + + for monitor_cfg in monitor_cfgs: + monitor_type = monitor_cfg["monitor_type"] + monitor_cls = BaseMonitor.get_monitor_cls(monitor_type) + monitor_ins = monitor_cls(monitor_cfg, context) + + self._monitor_list.append(monitor_ins) + + def start_monitors(self): + for _monotor_instace in self._monitor_list: + _monotor_instace.start_monitor() + + def wait_monitors(self): + for monitor in self._monitor_list: + monitor.wait_monitor() + + def verify_SLA(self): + sla_pass = True + for monitor in self._monitor_list: + sla_pass = sla_pass & monitor.verify_SLA() + return sla_pass diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py new file mode 100644 index 000000000..232340aca --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py @@ -0,0 +1,81 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import subprocess +import traceback + +import basemonitor as basemonitor + +LOG = logging.getLogger(__name__) + + +def _execute_shell_command(command): + '''execute shell script with error handling''' + exitcode = 0 + output = [] + try: + output = subprocess.check_output(command, shell=True) + except Exception: + exitcode = -1 + output = traceback.format_exc() + LOG.error("exec command '%s' error:\n " % command) + LOG.error(traceback.format_exc()) + + return exitcode, output + + +class MonitorOpenstackCmd(basemonitor.BaseMonitor): + """docstring for MonitorApi""" + + __monitor_type__ = "openstack-cmd" + + def monitor_func(self): + cmd = self._config["command_name"] + exit_status, stdout = _execute_shell_command(cmd) + if exit_status: + return False + return True + + def verify_SLA(self): + outage_time = self._result.get('outage_time', None) + LOG.debug("the _result:%s" % self._result) + max_outage_time = self._config["sla"]["max_outage_time"] + if outage_time > max_outage_time: + LOG.info("SLA failure: %f > %f" % (outage_time, max_outage_time)) + return False + else: + LOG.info("the sla is passed") + return True + + +def _test(): # pragma: no cover + host = { + "ip": "10.20.0.5", + "user": "root", + "key_filename": "/root/.ssh/id_rsa" + } + context = {"node1": host} + monitor_configs = [] + config = { + 'monitor_type': 'openstack-cmd', + 'command_name': 'nova image-list', + 'monitor_time': 1, + 'SLA': {'max_outage_time': 5} + } + monitor_configs.append(config) + + p = basemonitor.MonitorMgr() + p.init_monitors(monitor_configs, context) + p.start_monitors() + p.wait_monitors() + p.verify() + + +if __name__ == '__main__': # pragma: no cover + _test() diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml b/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml new file mode 100644 index 000000000..44f06038b --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml @@ -0,0 +1,9 @@ +--- +# sample config file for ha test +# +schema: "yardstick:task:0.1" + +kill-process: + inject_script: scripts/stop_service.bash + recovery_script: scripts/start_service.bash + check_script: scripts/check_service.bash diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py new file mode 100644 index 000000000..64e12f1dd --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py @@ -0,0 +1,84 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import yardstick.ssh as ssh + +import basemonitor as basemonitor + +LOG = logging.getLogger(__name__) + + +class MonitorProcess(basemonitor.BaseMonitor): + """docstring for MonitorApi""" + + __monitor_type__ = "process" + + def setup(self): + host = self._context[self._config["host"]] + ip = host.get("ip", None) + user = host.get("user", "root") + key_filename = host.get("key_filename", "~/.ssh/id_rsa") + + self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection.wait(timeout=600) + LOG.debug("ssh host success!") + self.check_script = self.get_script_fullpath( + "script_tools/check_service.bash") + self.process_name = self._config["process_name"] + + def monitor_func(self): + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0}".format(self.process_name), + stdin=open(self.check_script, "r")) + + if stdout and "running" in stdout: + LOG.info("check the envrioment success!") + return True + else: + LOG.error( + "the host envrioment is error, stdout:%s, stderr:%s" % + (stdout, stderr)) + return False + + def verify_SLA(self): + outage_time = self._result.get('outage_time', None) + max_outage_time = self._config["sla"]["max_recover_time"] + if outage_time > max_outage_time: + LOG.error("SLA failure: %f > %f" % (outage_time, max_outage_time)) + return False + else: + return True + + +def _test(): # pragma: no cover + host = { + "ip": "10.20.0.5", + "user": "root", + "key_filename": "/root/.ssh/id_rsa" + } + context = {"node1": host} + monitor_configs = [] + config = { + 'monitor_type': 'process', + 'process_name': 'nova-api', + 'host': "node1", + 'monitor_time': 1, + 'SLA': {'max_recover_time': 5} + } + monitor_configs.append(config) + + p = basemonitor.MonitorMgr() + p.init_monitors(monitor_configs, context) + p.start_monitors() + p.wait_monitors() + p.verify() + + +if __name__ == '__main__': # pragma: no cover + _test() diff --git a/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash b/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash new file mode 100755 index 000000000..cc898a859 --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash @@ -0,0 +1,18 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# check the status of a service + +set -e + +service_name=$1 + +service $service_name status |