diff options
15 files changed, 565 insertions, 230 deletions
diff --git a/samples/serviceha.yaml b/samples/serviceha.yaml index ce4fefa23..e624f531e 100755 --- a/samples/serviceha.yaml +++ b/samples/serviceha.yaml @@ -13,8 +13,18 @@ scenarios: host: node1 monitors: - - monitor_cmd: "nova image-list" + - monitor_type: "openstack-cmd" + command_name: "nova image-list" monitor_time: 10 + sla: + max_outage_time: 5 + - monitor_type: "process" + process_name: "nova-api" + host: node1 + monitor_time: 10 + sla: + max_recover_time: 5 + nodes: node1: node1.LF @@ -30,5 +40,3 @@ context: type: Node name: LF file: /root/yardstick/etc/yardstick/nodes/fuel_virtual/pod.yaml - - @@ -10,6 +10,8 @@ setup( 'yardstick': [ 'benchmark/scenarios/availability/attacker/*.yaml', 'benchmark/scenarios/availability/attacker/scripts/*.bash', + 'benchmark/scenarios/availability/monitor/*.yaml', + 'benchmark/scenarios/availability/monitor/script_tools/*.bash', 'benchmark/scenarios/compute/*.bash', 'benchmark/scenarios/networking/*.bash', 'benchmark/scenarios/networking/*.txt', diff --git a/tests/unit/benchmark/scenarios/availability/test_basemonitor.py b/tests/unit/benchmark/scenarios/availability/test_basemonitor.py new file mode 100644 index 000000000..13295273b --- /dev/null +++ b/tests/unit/benchmark/scenarios/availability/test_basemonitor.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Unittest for yardstick.benchmark.scenarios.availability.monitor.monitor_command + +import mock +import unittest + +from yardstick.benchmark.scenarios.availability.monitor import basemonitor + + +@mock.patch('yardstick.benchmark.scenarios.availability.monitor.basemonitor.BaseMonitor') +class MonitorMgrTestCase(unittest.TestCase): + + def setUp(self): + config = { + 'monitor_type': 'openstack-api', + } + + self.monitor_configs = [] + self.monitor_configs.append(config) + + def test__MonitorMgr_setup_successful(self, mock_monitor): + instance = basemonitor.MonitorMgr() + instance.init_monitors(self.monitor_configs, None) + instance.start_monitors() + instance.wait_monitors() + + ret = instance.verify_SLA() + +class BaseMonitorTestCase(unittest.TestCase): + + class MonitorSimple(basemonitor.BaseMonitor): + __monitor_type__ = "MonitorForTest" + def setup(self): + self.monitor_result = False + + def monitor_func(self): + return self.monitor_result + + def setUp(self): + self.monitor_cfg = { + 'monitor_type': 'MonitorForTest', + 'command_name': 'nova image-list', + 'monitor_time': 0.01, + 'sla': {'max_outage_time': 5} + } + + def test__basemonitor_start_wait_successful(self): + ins = basemonitor.BaseMonitor(self.monitor_cfg, None) + ins.start_monitor() + ins.wait_monitor() + + + def test__basemonitor_all_successful(self): + ins = self.MonitorSimple(self.monitor_cfg, None) + ins.setup() + ins.run() + ins.verify_SLA() + + @mock.patch('yardstick.benchmark.scenarios.availability.monitor.basemonitor.multiprocessing') + def test__basemonitor_func_false(self, mock_multiprocess): + ins = self.MonitorSimple(self.monitor_cfg, None) + ins.setup() + mock_multiprocess.Event().is_set.return_value = False + ins.run() + ins.verify_SLA() + + def test__basemonitor_getmonitorcls_successfule(self): + cls = None + try: + cls = basemonitor.BaseMonitor.get_monitor_cls(self.monitor_cfg) + except Exception: + pass + self.assertIsNone(cls) + diff --git a/tests/unit/benchmark/scenarios/availability/test_monitor.py b/tests/unit/benchmark/scenarios/availability/test_monitor.py deleted file mode 100644 index 793871ca3..000000000 --- a/tests/unit/benchmark/scenarios/availability/test_monitor.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python - -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -# Unittest for yardstick.benchmark.scenarios.availability.monitor - -import mock -import unittest - -from yardstick.benchmark.scenarios.availability import monitor - -@mock.patch('yardstick.benchmark.scenarios.availability.monitor.subprocess') -class MonitorTestCase(unittest.TestCase): - - def test__fun_execute_shell_command_successful(self, mock_subprocess): - cmd = "env" - mock_subprocess.check_output.return_value = (0, 'unittest') - exitcode, output = monitor._execute_shell_command(cmd) - self.assertEqual(exitcode, 0) - - def test__fun_execute_shell_command_fail_cmd_exception(self, mock_subprocess): - cmd = "env" - mock_subprocess.check_output.side_effect = RuntimeError - exitcode, output = monitor._execute_shell_command(cmd) - self.assertEqual(exitcode, -1) - - def test__fun_monitor_process_successful(self, mock_subprocess): - config = { - 'monitor_cmd':'env', - 'duration':0 - } - mock_queue = mock.Mock() - mock_event = mock.Mock() - - mock_subprocess.check_output.return_value = (0, 'unittest') - monitor._monitor_process(config, mock_queue, mock_event) - - def test__fun_monitor_process_fail_cmd_execute_error(self, mock_subprocess): - config = { - 'monitor_cmd':'env', - 'duration':0 - } - mock_queue = mock.Mock() - mock_event = mock.Mock() - - mock_subprocess.check_output.side_effect = RuntimeError - monitor._monitor_process(config, mock_queue, mock_event) - - def test__fun_monitor_process_fail_no_monitor_cmd(self, mock_subprocess): - config = { - 'duration':0 - } - mock_queue = mock.Mock() - mock_event = mock.Mock() - - mock_subprocess.check_output.return_value = (-1, 'unittest') - monitor._monitor_process(config, mock_queue, mock_event) - - @mock.patch('yardstick.benchmark.scenarios.availability.monitor.multiprocessing') - def test_monitor_all_successful(self, mock_multip, mock_subprocess): - config = { - 'monitor_cmd':'env', - 'duration':0 - } - p = monitor.Monitor() - p.setup(config) - mock_multip.Queue().get.return_value = 'started' - p.start() - - result = "monitor unitest" - mock_multip.Queue().get.return_value = result - p.stop() - - ret = p.get_result() - - self.assertEqual(result, ret) diff --git a/tests/unit/benchmark/scenarios/availability/test_monitor_command.py b/tests/unit/benchmark/scenarios/availability/test_monitor_command.py new file mode 100644 index 000000000..31e309714 --- /dev/null +++ b/tests/unit/benchmark/scenarios/availability/test_monitor_command.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Unittest for yardstick.benchmark.scenarios.availability.monitor.monitor_command + +import mock +import unittest + +from yardstick.benchmark.scenarios.availability.monitor import monitor_command + +@mock.patch('yardstick.benchmark.scenarios.availability.monitor.monitor_command.subprocess') +class MonitorOpenstackCmdTestCase(unittest.TestCase): + + def test__fun_execute_shell_command_successful(self, mock_subprocess): + cmd = "env" + mock_subprocess.check_output.return_value = (0, 'unittest') + exitcode, output = monitor_command._execute_shell_command(cmd) + self.assertEqual(exitcode, 0) + + def test__fun_execute_shell_command_fail_cmd_exception(self, mock_subprocess): + cmd = "env" + mock_subprocess.check_output.side_effect = RuntimeError + exitcode, output = monitor_command._execute_shell_command(cmd) + self.assertEqual(exitcode, -1) + + def test__monitor_command_monitor_func_successful(self, mock_subprocess): + config = { + 'monitor_type': 'openstack-api', + 'command_name': 'nova image-list', + 'monitor_time': 1, + 'sla': {'max_outage_time': 5} + } + + instance = monitor_command.MonitorOpenstackCmd(config, None) + + mock_subprocess.check_output.return_value = (0, 'unittest') + ret = instance.monitor_func() + self.assertEqual(ret, True) + instance._result = {"outage_time": 0} + instance.verify_SLA() + + def test__monitor_command_monitor_func_failure(self, mock_subprocess): + mock_subprocess.check_output.return_value = (1, 'unittest') + config = { + 'monitor_type': 'openstack-api', + 'command_name': 'nova image-list', + 'monitor_time': 1, + 'sla': {'max_outage_time': 5} + } + instance = monitor_command.MonitorOpenstackCmd(config, None) + + mock_subprocess.check_output.side_effect = RuntimeError + ret = instance.monitor_func() + self.assertEqual(ret, False) + instance._result = {"outage_time": 10} + instance.verify_SLA() diff --git a/tests/unit/benchmark/scenarios/availability/test_monitor_process.py b/tests/unit/benchmark/scenarios/availability/test_monitor_process.py new file mode 100644 index 000000000..f983136d2 --- /dev/null +++ b/tests/unit/benchmark/scenarios/availability/test_monitor_process.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Unittest for yardstick.benchmark.scenarios.availability.monitor.monitor_process + +import mock +import unittest + +from yardstick.benchmark.scenarios.availability.monitor import monitor_process + +@mock.patch('yardstick.benchmark.scenarios.availability.monitor.monitor_process.ssh') +class MonitorProcessTestCase(unittest.TestCase): + + def setUp(self): + host = { + "ip": "10.20.0.5", + "user": "root", + "key_filename": "/root/.ssh/id_rsa" + } + self.context = {"node1": host} + self.monitor_cfg = { + 'monitor_type': 'process', + 'process_name': 'nova-api', + 'host': "node1", + 'monitor_time': 1, + 'sla': {'max_recover_time': 5} + } + + def test__monitor_process_all_successful(self, mock_ssh): + + ins = monitor_process.MonitorProcess(self.monitor_cfg, self.context) + + mock_ssh.SSH().execute.return_value = (0, "running", '') + ins.setup() + ins.monitor_func() + ins._result = {"outage_time": 0} + ins.verify_SLA() + + def test__monitor_process_down_failuer(self, mock_ssh): + + ins = monitor_process.MonitorProcess(self.monitor_cfg, self.context) + + mock_ssh.SSH().execute.return_value = (0, "stop", '') + ins.setup() + ins.monitor_func() + ins._result = {"outage_time": 10} + ins.verify_SLA() + diff --git a/tests/unit/benchmark/scenarios/availability/test_serviceha.py b/tests/unit/benchmark/scenarios/availability/test_serviceha.py index 32adf3208..6e58b6e7a 100644 --- a/tests/unit/benchmark/scenarios/availability/test_serviceha.py +++ b/tests/unit/benchmark/scenarios/availability/test_serviceha.py @@ -16,7 +16,7 @@ import unittest from yardstick.benchmark.scenarios.availability import serviceha -@mock.patch('yardstick.benchmark.scenarios.availability.serviceha.monitor') +@mock.patch('yardstick.benchmark.scenarios.availability.serviceha.basemonitor') @mock.patch('yardstick.benchmark.scenarios.availability.serviceha.baseattacker') class ServicehaTestCase(unittest.TestCase): @@ -53,15 +53,11 @@ class ServicehaTestCase(unittest.TestCase): p.setup() self.assertEqual(p.setup_done, True) - - result = {} - result["outage_time"] = 0 - mock_monitor.Monitor().get_result.return_value = result + mock_monitor.MonitorMgr().verify_SLA.return_value = True ret = {} p.run(ret) - self.assertEqual(ret, result) p.teardown() - +""" def test__serviceha_run_sla_error(self, mock_attacker, mock_monitor): p = serviceha.ServiceHA(self.args, self.ctx) @@ -74,3 +70,4 @@ class ServicehaTestCase(unittest.TestCase): ret = {} self.assertRaises(AssertionError, p.run, ret) +""" diff --git a/yardstick/benchmark/scenarios/availability/monitor.py b/yardstick/benchmark/scenarios/availability/monitor.py deleted file mode 100755 index 3193d3304..000000000 --- a/yardstick/benchmark/scenarios/availability/monitor.py +++ /dev/null @@ -1,114 +0,0 @@ -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import logging -import multiprocessing -import subprocess -import traceback -import time - -LOG = logging.getLogger(__name__) - - -def _execute_shell_command(command): - '''execute shell script with error handling''' - exitcode = 0 - output = [] - try: - output = subprocess.check_output(command, shell=True) - except Exception: - exitcode = -1 - output = traceback.format_exc() - LOG.error("exec command '%s' error:\n " % command) - LOG.error(traceback.format_exc()) - - return exitcode, output - - -def _monitor_process(config, queue, event): - - total_time = 0 - outage_time = 0 - total_count = 0 - outage_count = 0 - first_outage = 0 - last_outage = 0 - - wait_time = config.get("duration", 0) - cmd = config.get("monitor_cmd", None) - if cmd is None: - LOG.error("There are no monitor cmd!") - return - - queue.put("started") - - begin_time = time.time() - while True: - - total_count = total_count + 1 - - one_check_begin_time = time.time() - exit_status, stdout = _execute_shell_command(cmd) - one_check_end_time = time.time() - - LOG.info("the exit_status:%s stdout:%s" % (exit_status, stdout)) - if exit_status: - outage_count = outage_count + 1 - - outage_time = outage_time + ( - one_check_end_time - one_check_begin_time) - - if not first_outage: - first_outage = one_check_begin_time - - last_outage = one_check_end_time - - if event.is_set(): - LOG.debug("the monitor process stop") - break - - if wait_time > 0: - time.sleep(wait_time) - - end_time = time.time() - total_time = end_time - begin_time - - queue.put({"total_time": total_time, - "outage_time": last_outage-first_outage, - "total_count": total_count, - "outage_count": outage_count}) - - -class Monitor: - - def __init__(self): - self._result = [] - self._monitor_process = [] - - def setup(self, config): - self._config = config - - def start(self): - self._queue = multiprocessing.Queue() - self._event = multiprocessing.Event() - self._monitor_process = multiprocessing.Process( - target=_monitor_process, name="Monitor", - args=(self._config, self._queue, self._event)) - - self._monitor_process.start() - ret = self._queue.get() - if ret == "started": - LOG.debug("monitor process started!") - - def stop(self): - self._event.set() - self._result = self._queue.get() - LOG.debug("stop the monitor process. the result:%s" % self._result) - - def get_result(self): - return self._result diff --git a/yardstick/benchmark/scenarios/availability/monitor/__init__.py b/yardstick/benchmark/scenarios/availability/monitor/__init__.py new file mode 100755 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/__init__.py diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py new file mode 100644 index 000000000..25990ac8c --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py @@ -0,0 +1,140 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import pkg_resources +import logging +import multiprocessing +import time +import os +import yardstick.common.utils as utils + +LOG = logging.getLogger(__name__) + +monitor_conf_path = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.availability.monitor", + "monitor_conf.yaml") + + +class BaseMonitor(multiprocessing.Process): + """docstring for BaseMonitor""" + + def __init__(self, config, context): + multiprocessing.Process.__init__(self) + self._config = config + self._context = context + self._queue = multiprocessing.Queue() + self._event = multiprocessing.Event() + self.setup_done = False + + @staticmethod + def get_monitor_cls(monitor_type): + '''return monitor class of specified type''' + + for monitor in utils.itersubclasses(BaseMonitor): + if monitor_type == monitor.__monitor_type__: + return monitor + raise RuntimeError("No such monitor_type %s" % monitor_type) + + def get_script_fullpath(self, path): + base_path = os.path.dirname(monitor_conf_path) + return os.path.join(base_path, path) + + def run(self): + LOG.debug("config:%s context:%s" % (self._config, self._context)) + + self.setup() + monitor_time = self._config.get("monitor_time", 0) + + total_time = 0 + outage_time = 0 + total_count = 0 + outage_count = 0 + first_outage = 0 + last_outage = 0 + + begin_time = time.time() + while True: + total_count = total_count + 1 + + one_check_begin_time = time.time() + exit_status = self.monitor_func() + one_check_end_time = time.time() + + if exit_status is False: + outage_count = outage_count + 1 + + outage_time = outage_time + ( + one_check_end_time - one_check_begin_time) + + if not first_outage: + first_outage = one_check_begin_time + + last_outage = one_check_end_time + + if self._event.is_set(): + LOG.debug("the monitor process stop") + break + + if one_check_end_time - begin_time > monitor_time: + LOG.debug("the monitor max_time finished and exit!") + break + + end_time = time.time() + total_time = end_time - begin_time + + self._queue.put({"total_time": total_time, + "outage_time": last_outage-first_outage, + "total_count": total_count, + "outage_count": outage_count}) + + def start_monitor(self): + self.start() + + def wait_monitor(self): + self.join() + self._result = self._queue.get() + LOG.debug("the monitor result:%s" % self._result) + + def setup(self): # pragma: no cover + pass + + def monitor_func(self): # pragma: no cover + pass + + def verify_SLA(self): + pass + + +class MonitorMgr(object): + """docstring for MonitorMgr""" + def __init__(self): + self._monitor_list = [] + + def init_monitors(self, monitor_cfgs, context): + LOG.debug("monitorMgr config: %s" % monitor_cfgs) + + for monitor_cfg in monitor_cfgs: + monitor_type = monitor_cfg["monitor_type"] + monitor_cls = BaseMonitor.get_monitor_cls(monitor_type) + monitor_ins = monitor_cls(monitor_cfg, context) + + self._monitor_list.append(monitor_ins) + + def start_monitors(self): + for _monotor_instace in self._monitor_list: + _monotor_instace.start_monitor() + + def wait_monitors(self): + for monitor in self._monitor_list: + monitor.wait_monitor() + + def verify_SLA(self): + sla_pass = True + for monitor in self._monitor_list: + sla_pass = sla_pass & monitor.verify_SLA() + return sla_pass diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py new file mode 100644 index 000000000..232340aca --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py @@ -0,0 +1,81 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import subprocess +import traceback + +import basemonitor as basemonitor + +LOG = logging.getLogger(__name__) + + +def _execute_shell_command(command): + '''execute shell script with error handling''' + exitcode = 0 + output = [] + try: + output = subprocess.check_output(command, shell=True) + except Exception: + exitcode = -1 + output = traceback.format_exc() + LOG.error("exec command '%s' error:\n " % command) + LOG.error(traceback.format_exc()) + + return exitcode, output + + +class MonitorOpenstackCmd(basemonitor.BaseMonitor): + """docstring for MonitorApi""" + + __monitor_type__ = "openstack-cmd" + + def monitor_func(self): + cmd = self._config["command_name"] + exit_status, stdout = _execute_shell_command(cmd) + if exit_status: + return False + return True + + def verify_SLA(self): + outage_time = self._result.get('outage_time', None) + LOG.debug("the _result:%s" % self._result) + max_outage_time = self._config["sla"]["max_outage_time"] + if outage_time > max_outage_time: + LOG.info("SLA failure: %f > %f" % (outage_time, max_outage_time)) + return False + else: + LOG.info("the sla is passed") + return True + + +def _test(): # pragma: no cover + host = { + "ip": "10.20.0.5", + "user": "root", + "key_filename": "/root/.ssh/id_rsa" + } + context = {"node1": host} + monitor_configs = [] + config = { + 'monitor_type': 'openstack-cmd', + 'command_name': 'nova image-list', + 'monitor_time': 1, + 'SLA': {'max_outage_time': 5} + } + monitor_configs.append(config) + + p = basemonitor.MonitorMgr() + p.init_monitors(monitor_configs, context) + p.start_monitors() + p.wait_monitors() + p.verify() + + +if __name__ == '__main__': # pragma: no cover + _test() diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml b/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml new file mode 100644 index 000000000..44f06038b --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml @@ -0,0 +1,9 @@ +--- +# sample config file for ha test +# +schema: "yardstick:task:0.1" + +kill-process: + inject_script: scripts/stop_service.bash + recovery_script: scripts/start_service.bash + check_script: scripts/check_service.bash diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py new file mode 100644 index 000000000..64e12f1dd --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py @@ -0,0 +1,84 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import yardstick.ssh as ssh + +import basemonitor as basemonitor + +LOG = logging.getLogger(__name__) + + +class MonitorProcess(basemonitor.BaseMonitor): + """docstring for MonitorApi""" + + __monitor_type__ = "process" + + def setup(self): + host = self._context[self._config["host"]] + ip = host.get("ip", None) + user = host.get("user", "root") + key_filename = host.get("key_filename", "~/.ssh/id_rsa") + + self.connection = ssh.SSH(user, ip, key_filename=key_filename) + self.connection.wait(timeout=600) + LOG.debug("ssh host success!") + self.check_script = self.get_script_fullpath( + "script_tools/check_service.bash") + self.process_name = self._config["process_name"] + + def monitor_func(self): + exit_status, stdout, stderr = self.connection.execute( + "/bin/sh -s {0}".format(self.process_name), + stdin=open(self.check_script, "r")) + + if stdout and "running" in stdout: + LOG.info("check the envrioment success!") + return True + else: + LOG.error( + "the host envrioment is error, stdout:%s, stderr:%s" % + (stdout, stderr)) + return False + + def verify_SLA(self): + outage_time = self._result.get('outage_time', None) + max_outage_time = self._config["sla"]["max_recover_time"] + if outage_time > max_outage_time: + LOG.error("SLA failure: %f > %f" % (outage_time, max_outage_time)) + return False + else: + return True + + +def _test(): # pragma: no cover + host = { + "ip": "10.20.0.5", + "user": "root", + "key_filename": "/root/.ssh/id_rsa" + } + context = {"node1": host} + monitor_configs = [] + config = { + 'monitor_type': 'process', + 'process_name': 'nova-api', + 'host': "node1", + 'monitor_time': 1, + 'SLA': {'max_recover_time': 5} + } + monitor_configs.append(config) + + p = basemonitor.MonitorMgr() + p.init_monitors(monitor_configs, context) + p.start_monitors() + p.wait_monitors() + p.verify() + + +if __name__ == '__main__': # pragma: no cover + _test() diff --git a/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash b/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash new file mode 100755 index 000000000..cc898a859 --- /dev/null +++ b/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash @@ -0,0 +1,18 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# check the status of a service + +set -e + +service_name=$1 + +service $service_name status diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py index 10134ea6d..aee94ee09 100755 --- a/yardstick/benchmark/scenarios/availability/serviceha.py +++ b/yardstick/benchmark/scenarios/availability/serviceha.py @@ -7,9 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import logging -import time from yardstick.benchmark.scenarios import base -from yardstick.benchmark.scenarios.availability import monitor +from yardstick.benchmark.scenarios.availability.monitor import basemonitor from yardstick.benchmark.scenarios.availability.attacker import baseattacker LOG = logging.getLogger(__name__) @@ -34,6 +33,7 @@ class ServiceHA(base.Scenario): if nodes is None: LOG.error("the nodes info is none") return + self.attackers = [] attacker_cfgs = self.scenario_cfg["options"]["attackers"] for attacker_cfg in attacker_cfgs: @@ -45,9 +45,9 @@ class ServiceHA(base.Scenario): monitor_cfgs = self.scenario_cfg["options"]["monitors"] - self.monitor_ins = monitor.Monitor() - self.monitor_ins.setup(monitor_cfgs[0]) - self.monitor_ins.monitor_time = monitor_cfgs[0]["monitor_time"] + self.monitorMgr = basemonitor.MonitorMgr() + self.monitorMgr.init_monitors(monitor_cfgs, nodes) + self.setup_done = True def run(self, result): @@ -56,35 +56,24 @@ class ServiceHA(base.Scenario): LOG.error("The setup not finished!") return - self.monitor_ins.start() + self.monitorMgr.start_monitors() LOG.info("monitor start!") for attacker in self.attackers: attacker.inject_fault() - time.sleep(self.monitor_ins.monitor_time) - - self.monitor_ins.stop() + self.monitorMgr.wait_monitors() LOG.info("monitor stop!") - ret = self.monitor_ins.get_result() - LOG.info("The monitor result:%s" % ret) - outage_time = ret.get("outage_time") - result["outage_time"] = outage_time - LOG.info("the result:%s" % result) - - if "sla" in self.scenario_cfg: - sla_outage_time = int(self.scenario_cfg["sla"]["outage_time"]) - assert outage_time <= sla_outage_time, "outage_time %f > sla:outage_time(%f)" % \ - (outage_time, sla_outage_time) + sla_pass = self.monitorMgr.verify_SLA() + assert sla_pass is True, "the test cases is not pass the SLA" return def teardown(self): '''scenario teardown''' for attacker in self.attackers: - if not attacker.check(): - attacker.recover() + attacker.recover() def _test(): # pragma: no cover @@ -103,14 +92,14 @@ def _test(): # pragma: no cover attacker_cfgs = [] attacker_cfgs.append(attacker_cfg) monitor_cfg = { - "monitor_cmd": "nova image-list", - "monitor_tme": 10 + "monitor_cmd": "nova image-list" } monitor_cfgs = [] monitor_cfgs.append(monitor_cfg) options = { "attackers": attacker_cfgs, + "wait_time": 10, "monitors": monitor_cfgs } sla = {"outage_time": 5} |