From 6a5d8a6d58ab501184313eda84820294ff3597e7 Mon Sep 17 00:00:00 2001 From: wym_libra Date: Mon, 26 Oct 2015 07:54:31 +0000 Subject: A initial HA test case 1)stop an openstack service 2)then monitor the corresponding api and check the availability of it 3)recovery the openstack service JIRA: YARDSTICK-149 Change-Id: Id7b77d2f5c71844729c04f37442c8cfaa270ab12 Signed-off-by: wym_libra --- .../benchmark/scenarios/availability/__init__.py | 0 .../availability/ha_tools/check_service.bash | 18 ++ .../scenarios/availability/ha_tools/ha_conf.yaml | 12 ++ .../availability/ha_tools/start_service.bash | 18 ++ .../availability/ha_tools/stop_service.bash | 21 +++ .../benchmark/scenarios/availability/monitor.py | 114 ++++++++++++ .../benchmark/scenarios/availability/serviceha.py | 193 +++++++++++++++++++++ 7 files changed, 376 insertions(+) create mode 100755 yardstick/benchmark/scenarios/availability/__init__.py create mode 100755 yardstick/benchmark/scenarios/availability/ha_tools/check_service.bash create mode 100644 yardstick/benchmark/scenarios/availability/ha_tools/ha_conf.yaml create mode 100755 yardstick/benchmark/scenarios/availability/ha_tools/start_service.bash create mode 100755 yardstick/benchmark/scenarios/availability/ha_tools/stop_service.bash create mode 100755 yardstick/benchmark/scenarios/availability/monitor.py create mode 100755 yardstick/benchmark/scenarios/availability/serviceha.py (limited to 'yardstick/benchmark') diff --git a/yardstick/benchmark/scenarios/availability/__init__.py b/yardstick/benchmark/scenarios/availability/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/check_service.bash b/yardstick/benchmark/scenarios/availability/ha_tools/check_service.bash new file mode 100755 index 000000000..cc898a859 --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/ha_tools/check_service.bash @@ -0,0 +1,18 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# check the status of a service + +set -e + +service_name=$1 + +service $service_name status diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/ha_conf.yaml b/yardstick/benchmark/scenarios/availability/ha_tools/ha_conf.yaml new file mode 100644 index 000000000..67e56eb4f --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/ha_tools/ha_conf.yaml @@ -0,0 +1,12 @@ +--- +# sample config file for ha test +# +schema: "yardstick:task:0.1" + +nova-api: +- + type: stop-service + inject_script: ha_tools/stop_service.bash + recovery_script: ha_tools/start_service.bash + check_script: ha_tools/check_service.bash + monitor_cmd: nova image-list diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/start_service.bash b/yardstick/benchmark/scenarios/availability/ha_tools/start_service.bash new file mode 100755 index 000000000..c1bf8b7eb --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/ha_tools/start_service.bash @@ -0,0 +1,18 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Start a service and check the service is started + +set -e + +service_name=$1 + +service $service_name start diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/stop_service.bash b/yardstick/benchmark/scenarios/availability/ha_tools/stop_service.bash new file mode 100755 index 000000000..a8901784e --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/ha_tools/stop_service.bash @@ -0,0 +1,21 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Stop a service and check the service is stoped + +set -e + +service_name=$1 + +service $service_name stop + +# TODO +# check the service status diff --git a/yardstick/benchmark/scenarios/availability/monitor.py b/yardstick/benchmark/scenarios/availability/monitor.py new file mode 100755 index 000000000..3193d3304 --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor.py @@ -0,0 +1,114 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import multiprocessing +import subprocess +import traceback +import time + +LOG = logging.getLogger(__name__) + + +def _execute_shell_command(command): + '''execute shell script with error handling''' + exitcode = 0 + output = [] + try: + output = subprocess.check_output(command, shell=True) + except Exception: + exitcode = -1 + output = traceback.format_exc() + LOG.error("exec command '%s' error:\n " % command) + LOG.error(traceback.format_exc()) + + return exitcode, output + + +def _monitor_process(config, queue, event): + + total_time = 0 + outage_time = 0 + total_count = 0 + outage_count = 0 + first_outage = 0 + last_outage = 0 + + wait_time = config.get("duration", 0) + cmd = config.get("monitor_cmd", None) + if cmd is None: + LOG.error("There are no monitor cmd!") + return + + queue.put("started") + + begin_time = time.time() + while True: + + total_count = total_count + 1 + + one_check_begin_time = time.time() + exit_status, stdout = _execute_shell_command(cmd) + one_check_end_time = time.time() + + LOG.info("the exit_status:%s stdout:%s" % (exit_status, stdout)) + if exit_status: + outage_count = outage_count + 1 + + outage_time = outage_time + ( + one_check_end_time - one_check_begin_time) + + if not first_outage: + first_outage = one_check_begin_time + + last_outage = one_check_end_time + + if event.is_set(): + LOG.debug("the monitor process stop") + break + + if wait_time > 0: + time.sleep(wait_time) + + end_time = time.time() + total_time = end_time - begin_time + + queue.put({"total_time": total_time, + "outage_time": last_outage-first_outage, + "total_count": total_count, + "outage_count": outage_count}) + + +class Monitor: + + def __init__(self): + self._result = [] + self._monitor_process = [] + + def setup(self, config): + self._config = config + + def start(self): + self._queue = multiprocessing.Queue() + self._event = multiprocessing.Event() + self._monitor_process = multiprocessing.Process( + target=_monitor_process, name="Monitor", + args=(self._config, self._queue, self._event)) + + self._monitor_process.start() + ret = self._queue.get() + if ret == "started": + LOG.debug("monitor process started!") + + def stop(self): + self._event.set() + self._result = self._queue.get() + LOG.debug("stop the monitor process. the result:%s" % self._result) + + def get_result(self): + return self._result diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py new file mode 100755 index 000000000..3e03e1da5 --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/serviceha.py @@ -0,0 +1,193 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import pkg_resources +import logging +import time +import yaml +import yardstick.ssh as ssh +from yardstick.benchmark.scenarios import base +from yardstick.benchmark.scenarios.availability import monitor + +LOG = logging.getLogger(__name__) + + +class ServiceHA(base.Scenario): + """TODO: docstring of ServiceHA + """ + __scenario_type__ = "ServiceHA" + + HA_CONF = "ha_tools/ha_conf.yaml" + + def __init__(self, scenario_cfg, context_cfg): + self.scenario_cfg = scenario_cfg + self.context_cfg = context_cfg + self.service_name = scenario_cfg["options"]["component"] + self.fault_type = scenario_cfg["options"]["fault_type"] + self.fault_time = scenario_cfg["options"].get("fault_time", 0) + self.fault_cfg = None + self.setup_done = False + self.need_teardown = False + + def setup(self): + '''scenario setup''' + self.ha_conf_file = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.availability", + ServiceHA.HA_CONF) + ha_cfg = [] + with open(self.ha_conf_file) as stream: + ha_cfg = yaml.load(stream) + LOG.debug("ha_cfg content:%s" % ha_cfg) + + # check the ha_conf contains the service defined in test cases yaml + service_cfg = ha_cfg.get(self.service_name, None) + if not service_cfg: + LOG.error( + "The component %s can not be supported!" % self.service_name) + return + + for fault in service_cfg: + if fault["type"] == self.fault_type: + self.fault_cfg = fault + break + if not self.fault_cfg: + LOG.error( + "The fualt_type %s can not be supproted!" % self.fault_type) + return + LOG.debug("the fault_cfg :%s" % self.fault_cfg) + + self.fault_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.availability", + self.fault_cfg["inject_script"]) + self.recovery_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.availability", + self.fault_cfg["recovery_script"]) + self.check_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.availability", + self.fault_cfg["check_script"]) + + host = self.context_cfg.get("host", None) + ip = host.get("ip", None) + user = host.get("user", "root") + key_filename = host.get("key_filename", "~/.ssh/id_rsa") + LOG.info("The host: %s the service: %s" % (ip, self.service_name)) + LOG.debug("The params, host:%s fault_cfg:%s" % (host, self.fault_cfg)) + + LOG.debug( + "ssh connection ip:%s, user:%s, key_file:%s", + ip, user, key_filename) + self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection.wait(timeout=600) + LOG.debug("ssh host success!") + + # check the host envrioment + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0}".format(self.service_name), + stdin=open(self.check_script, "r")) + LOG.info( + "the exit_status:%s stdout:%s stderr:%s" % + (exit_status, stdout, stderr)) + if exit_status: + raise RuntimeError(stderr) + + if stdout and "running" in stdout: + LOG.info("check the envrioment success!") + else: + LOG.error( + "the host envrioment is error, stdout:%s, stderr:%s" % + (stdout, stderr)) + return + + self.setup_done = True + + def run(self, result): + """execute the benchmark""" + if not self.setup_done: + LOG.error("The setup not finished!") + return + + monitorInstance = monitor.Monitor() + monitorInstance.setup(self.fault_cfg) + monitorInstance.start() + LOG.info("monitor start!") + + LOG.info("Inject fault!") + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0}".format(self.service_name), + stdin=open(self.fault_script, "r")) + + if exit_status != 0: + monitorInstance.stop() + raise RuntimeError(stderr) + + self.need_teardown = True + time.sleep(self.fault_time) + + monitorInstance.stop() + LOG.info("monitor stop!") + + ret = monitorInstance.get_result() + LOG.info("The monitor result:%s" % ret) + outage_time = ret.get("outage_time") + result["outage_time"] = outage_time + LOG.info("the result:%s" % result) + + if "sla" in self.scenario_cfg: + sla_outage_time = int(self.scenario_cfg["sla"]["outage_time"]) + assert outage_time <= sla_outage_time, "outage_time %f > sla:outage_time(%f)" % \ + (outage_time, sla_outage_time) + + return + + def teardown(self): + '''scenario teardown''' + LOG.info("recory the everiment!") + + if self.need_teardown: + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0} ".format(self.service_name), + stdin=open(self.recovery_script, "r")) + + if exit_status: + raise RuntimeError(stderr) + else: + self.need_teardown = False + +""" +def _test(): + '''internal test function''' + host = { + "ip": "10.20.0.5", + "user": "root", + "key_filename": "/root/.ssh/id_rsa" + } + ctx = {"host": host} + + logger = logging.getLogger("yardstick") + logger.setLevel(logging.DEBUG) + + options = { + "component": "nova-api", + "fault_type": "stop-service" + } + sla = {"outage_time": 5} + args = {"options": options, "sla": sla} + + print "create instance" + terstInstance = ServiceHA(args, ctx) + + terstInstance.setup() + result = {} + terstInstance.run(result) + print result + + terstInstance.teardown() + +if __name__ == '__main__': + _test() +""" -- cgit 1.2.3-korg