Rewrite the HA test case (2)

idea: refact the Monitor class in old file "monitor.py" with the base
class and sub-class.
detail:
1) the BaseMonitor is the base class of other monitor
2) each monitor run in independent process
3) there are two monitor("openstack-cmd" and "process") for the first test case
4) MonitorMgr class used to manager monitor process

JIRA: YARDSTICK-149

Change-Id: I2eede94481f740812212e6cb673d175b5f543c15
Signed-off-by: wym_libra <yimin.wang@huawei.com>

15 files changed, 565 insertions, 230 deletions

diff --git a/samples/serviceha.yaml b/samples/serviceha.yaml
index ce4fefa23..e624f531e 100755
--- a/samples/serviceha.yaml
+++ b/samples/serviceha.yaml
@@ -13,8 +13,18 @@ scenarios:
       host: node1
 
     monitors:
-    - monitor_cmd: "nova image-list"
+    - monitor_type: "openstack-cmd"
+      command_name: "nova image-list"
       monitor_time: 10
+      sla:
+        max_outage_time: 5
+    - monitor_type: "process"
+      process_name: "nova-api"
+      host: node1
+      monitor_time: 10
+      sla:
+        max_recover_time: 5
+
   nodes:
     node1: node1.LF
 
@@ -30,5 +40,3 @@ context:
   type: Node
   name: LF
   file: /root/yardstick/etc/yardstick/nodes/fuel_virtual/pod.yaml
-
-
diff --git a/setup.py b/setup.py
index e923b23c5..3ef09d111 100755
--- a/setup.py
+++ b/setup.py
@@ -10,6 +10,8 @@ setup(
         'yardstick': [
             'benchmark/scenarios/availability/attacker/*.yaml',
             'benchmark/scenarios/availability/attacker/scripts/*.bash',
+            'benchmark/scenarios/availability/monitor/*.yaml',
+            'benchmark/scenarios/availability/monitor/script_tools/*.bash',
             'benchmark/scenarios/compute/*.bash',
             'benchmark/scenarios/networking/*.bash',
             'benchmark/scenarios/networking/*.txt',
diff --git a/tests/unit/benchmark/scenarios/availability/test_basemonitor.py b/tests/unit/benchmark/scenarios/availability/test_basemonitor.py
new file mode 100644
index 000000000..13295273b
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/availability/test_basemonitor.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.availability.monitor.monitor_command
+
+import mock
+import unittest
+
+from yardstick.benchmark.scenarios.availability.monitor import basemonitor
+
+
+@mock.patch('yardstick.benchmark.scenarios.availability.monitor.basemonitor.BaseMonitor')
+class MonitorMgrTestCase(unittest.TestCase):
+
+    def setUp(self):
+        config = {
+            'monitor_type': 'openstack-api',
+        }
+
+        self.monitor_configs = []
+        self.monitor_configs.append(config)
+
+    def test__MonitorMgr_setup_successful(self, mock_monitor):
+        instance = basemonitor.MonitorMgr()
+        instance.init_monitors(self.monitor_configs, None)
+        instance.start_monitors()
+        instance.wait_monitors()
+
+        ret = instance.verify_SLA()
+
+class BaseMonitorTestCase(unittest.TestCase):
+
+    class MonitorSimple(basemonitor.BaseMonitor):
+        __monitor_type__ = "MonitorForTest"
+        def setup(self):
+            self.monitor_result = False
+
+        def monitor_func(self):
+            return self.monitor_result
+
+    def setUp(self):
+        self.monitor_cfg = {
+            'monitor_type': 'MonitorForTest',
+            'command_name': 'nova image-list',
+            'monitor_time': 0.01,
+            'sla': {'max_outage_time': 5}
+        }
+
+    def test__basemonitor_start_wait_successful(self):
+        ins = basemonitor.BaseMonitor(self.monitor_cfg, None)
+        ins.start_monitor()
+        ins.wait_monitor()
+
+
+    def test__basemonitor_all_successful(self):
+        ins = self.MonitorSimple(self.monitor_cfg, None)
+        ins.setup()
+        ins.run()
+        ins.verify_SLA()
+
+    @mock.patch('yardstick.benchmark.scenarios.availability.monitor.basemonitor.multiprocessing')
+    def test__basemonitor_func_false(self, mock_multiprocess):
+        ins = self.MonitorSimple(self.monitor_cfg, None)
+        ins.setup()
+        mock_multiprocess.Event().is_set.return_value = False
+        ins.run()
+        ins.verify_SLA()
+
+    def test__basemonitor_getmonitorcls_successfule(self):
+        cls = None
+        try:
+            cls = basemonitor.BaseMonitor.get_monitor_cls(self.monitor_cfg)
+        except Exception:
+            pass
+        self.assertIsNone(cls)
+
diff --git a/tests/unit/benchmark/scenarios/availability/test_monitor.py b/tests/unit/benchmark/scenarios/availability/test_monitor.py
deleted file mode 100644
index 793871ca3..000000000
--- a/tests/unit/benchmark/scenarios/availability/test_monitor.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env python
-
-##############################################################################
-# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-# Unittest for yardstick.benchmark.scenarios.availability.monitor
-
-import mock
-import unittest
-
-from yardstick.benchmark.scenarios.availability import monitor
-
-@mock.patch('yardstick.benchmark.scenarios.availability.monitor.subprocess')
-class MonitorTestCase(unittest.TestCase):
-
-    def test__fun_execute_shell_command_successful(self, mock_subprocess):
-        cmd = "env"
-        mock_subprocess.check_output.return_value = (0, 'unittest')
-        exitcode, output = monitor._execute_shell_command(cmd)
-        self.assertEqual(exitcode, 0)
-
-    def test__fun_execute_shell_command_fail_cmd_exception(self, mock_subprocess):
-        cmd = "env"
-        mock_subprocess.check_output.side_effect = RuntimeError
-        exitcode, output = monitor._execute_shell_command(cmd)
-        self.assertEqual(exitcode, -1)
-
-    def test__fun_monitor_process_successful(self, mock_subprocess):
-        config = {
-            'monitor_cmd':'env',
-            'duration':0
-        }
-        mock_queue = mock.Mock()
-        mock_event = mock.Mock()
-
-        mock_subprocess.check_output.return_value = (0, 'unittest')
-        monitor._monitor_process(config, mock_queue, mock_event)
-
-    def test__fun_monitor_process_fail_cmd_execute_error(self, mock_subprocess):
-        config = {
-            'monitor_cmd':'env',
-            'duration':0
-        }
-        mock_queue = mock.Mock()
-        mock_event = mock.Mock()
-
-        mock_subprocess.check_output.side_effect = RuntimeError
-        monitor._monitor_process(config, mock_queue, mock_event)
-
-    def test__fun_monitor_process_fail_no_monitor_cmd(self, mock_subprocess):
-        config = {
-            'duration':0
-        }
-        mock_queue = mock.Mock()
-        mock_event = mock.Mock()
-
-        mock_subprocess.check_output.return_value = (-1, 'unittest')
-        monitor._monitor_process(config, mock_queue, mock_event)
-
-    @mock.patch('yardstick.benchmark.scenarios.availability.monitor.multiprocessing')
-    def test_monitor_all_successful(self, mock_multip, mock_subprocess):
-        config = {
-            'monitor_cmd':'env',
-            'duration':0
-        }
-        p = monitor.Monitor()
-        p.setup(config)
-        mock_multip.Queue().get.return_value = 'started'
-        p.start()
-
-        result = "monitor unitest"
-        mock_multip.Queue().get.return_value = result
-        p.stop()
-
-        ret = p.get_result()
-
-        self.assertEqual(result, ret)
diff --git a/tests/unit/benchmark/scenarios/availability/test_monitor_command.py b/tests/unit/benchmark/scenarios/availability/test_monitor_command.py
new file mode 100644
index 000000000..31e309714
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/availability/test_monitor_command.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.availability.monitor.monitor_command
+
+import mock
+import unittest
+
+from yardstick.benchmark.scenarios.availability.monitor import monitor_command
+
+@mock.patch('yardstick.benchmark.scenarios.availability.monitor.monitor_command.subprocess')
+class MonitorOpenstackCmdTestCase(unittest.TestCase):
+
+    def test__fun_execute_shell_command_successful(self, mock_subprocess):
+        cmd = "env"
+        mock_subprocess.check_output.return_value = (0, 'unittest')
+        exitcode, output = monitor_command._execute_shell_command(cmd)
+        self.assertEqual(exitcode, 0)
+
+    def test__fun_execute_shell_command_fail_cmd_exception(self, mock_subprocess):
+        cmd = "env"
+        mock_subprocess.check_output.side_effect = RuntimeError
+        exitcode, output = monitor_command._execute_shell_command(cmd)
+        self.assertEqual(exitcode, -1)
+
+    def test__monitor_command_monitor_func_successful(self, mock_subprocess):
+        config = {
+            'monitor_type': 'openstack-api',
+            'command_name': 'nova image-list',
+            'monitor_time': 1,
+            'sla': {'max_outage_time': 5}
+        }
+
+        instance = monitor_command.MonitorOpenstackCmd(config, None)
+
+        mock_subprocess.check_output.return_value = (0, 'unittest')
+        ret = instance.monitor_func()
+        self.assertEqual(ret, True)
+        instance._result = {"outage_time": 0}
+        instance.verify_SLA()
+
+    def test__monitor_command_monitor_func_failure(self, mock_subprocess):
+        mock_subprocess.check_output.return_value = (1, 'unittest')
+        config = {
+            'monitor_type': 'openstack-api',
+            'command_name': 'nova image-list',
+            'monitor_time': 1,
+            'sla': {'max_outage_time': 5}
+        }
+        instance = monitor_command.MonitorOpenstackCmd(config, None)
+
+        mock_subprocess.check_output.side_effect = RuntimeError
+        ret = instance.monitor_func()
+        self.assertEqual(ret, False)
+        instance._result = {"outage_time": 10}
+        instance.verify_SLA()
diff --git a/tests/unit/benchmark/scenarios/availability/test_monitor_process.py b/tests/unit/benchmark/scenarios/availability/test_monitor_process.py
new file mode 100644
index 000000000..f983136d2
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/availability/test_monitor_process.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.availability.monitor.monitor_process
+
+import mock
+import unittest
+
+from yardstick.benchmark.scenarios.availability.monitor import monitor_process
+
+@mock.patch('yardstick.benchmark.scenarios.availability.monitor.monitor_process.ssh')
+class MonitorProcessTestCase(unittest.TestCase):
+
+    def setUp(self):
+        host = {
+            "ip": "10.20.0.5",
+            "user": "root",
+            "key_filename": "/root/.ssh/id_rsa"
+        }
+        self.context = {"node1": host}
+        self.monitor_cfg = {
+            'monitor_type': 'process',
+            'process_name': 'nova-api',
+            'host': "node1",
+            'monitor_time': 1,
+            'sla': {'max_recover_time': 5}
+        }
+
+    def test__monitor_process_all_successful(self, mock_ssh):
+
+        ins = monitor_process.MonitorProcess(self.monitor_cfg, self.context)
+
+        mock_ssh.SSH().execute.return_value = (0, "running", '')
+        ins.setup()
+        ins.monitor_func()
+        ins._result = {"outage_time": 0}
+        ins.verify_SLA()
+
+    def test__monitor_process_down_failuer(self, mock_ssh):
+
+        ins = monitor_process.MonitorProcess(self.monitor_cfg, self.context)
+
+        mock_ssh.SSH().execute.return_value = (0, "stop", '')
+        ins.setup()
+        ins.monitor_func()
+        ins._result = {"outage_time": 10}
+        ins.verify_SLA()
+
diff --git a/tests/unit/benchmark/scenarios/availability/test_serviceha.py b/tests/unit/benchmark/scenarios/availability/test_serviceha.py
index 32adf3208..6e58b6e7a 100644
--- a/tests/unit/benchmark/scenarios/availability/test_serviceha.py
+++ b/tests/unit/benchmark/scenarios/availability/test_serviceha.py
@@ -16,7 +16,7 @@ import unittest
 
 from yardstick.benchmark.scenarios.availability import serviceha
 
-@mock.patch('yardstick.benchmark.scenarios.availability.serviceha.monitor')
+@mock.patch('yardstick.benchmark.scenarios.availability.serviceha.basemonitor')
 @mock.patch('yardstick.benchmark.scenarios.availability.serviceha.baseattacker')
 class ServicehaTestCase(unittest.TestCase):
 
@@ -53,15 +53,11 @@ class ServicehaTestCase(unittest.TestCase):
 
         p.setup()
         self.assertEqual(p.setup_done, True)
-
-        result = {}
-        result["outage_time"] = 0
-        mock_monitor.Monitor().get_result.return_value = result
+        mock_monitor.MonitorMgr().verify_SLA.return_value = True
         ret = {}
         p.run(ret)
-        self.assertEqual(ret, result)
         p.teardown()
-
+"""
     def test__serviceha_run_sla_error(self, mock_attacker, mock_monitor):
         p = serviceha.ServiceHA(self.args, self.ctx)
 
@@ -74,3 +70,4 @@ class ServicehaTestCase(unittest.TestCase):
 
         ret = {}
         self.assertRaises(AssertionError, p.run, ret)
+"""
diff --git a/yardstick/benchmark/scenarios/availability/monitor.py b/yardstick/benchmark/scenarios/availability/monitor.py
deleted file mode 100755
index 3193d3304..000000000
--- a/yardstick/benchmark/scenarios/availability/monitor.py
+++ /dev/null
@@ -1,114 +0,0 @@
-##############################################################################
-# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-import logging
-import multiprocessing
-import subprocess
-import traceback
-import time
-
-LOG = logging.getLogger(__name__)
-
-
-def _execute_shell_command(command):
-    '''execute shell script with error handling'''
-    exitcode = 0
-    output = []
-    try:
-        output = subprocess.check_output(command, shell=True)
-    except Exception:
-        exitcode = -1
-        output = traceback.format_exc()
-        LOG.error("exec command '%s' error:\n " % command)
-        LOG.error(traceback.format_exc())
-
-    return exitcode, output
-
-
-def _monitor_process(config, queue, event):
-
-    total_time = 0
-    outage_time = 0
-    total_count = 0
-    outage_count = 0
-    first_outage = 0
-    last_outage = 0
-
-    wait_time = config.get("duration", 0)
-    cmd = config.get("monitor_cmd", None)
-    if cmd is None:
-        LOG.error("There are no monitor cmd!")
-        return
-
-    queue.put("started")
-
-    begin_time = time.time()
-    while True:
-
-        total_count = total_count + 1
-
-        one_check_begin_time = time.time()
-        exit_status, stdout = _execute_shell_command(cmd)
-        one_check_end_time = time.time()
-
-        LOG.info("the exit_status:%s stdout:%s" % (exit_status, stdout))
-        if exit_status:
-            outage_count = outage_count + 1
-
-            outage_time = outage_time + (
-                one_check_end_time - one_check_begin_time)
-
-            if not first_outage:
-                first_outage = one_check_begin_time
-
-            last_outage = one_check_end_time
-
-        if event.is_set():
-            LOG.debug("the monitor process stop")
-            break
-
-        if wait_time > 0:
-            time.sleep(wait_time)
-
-    end_time = time.time()
-    total_time = end_time - begin_time
-
-    queue.put({"total_time": total_time,
-               "outage_time": last_outage-first_outage,
-               "total_count": total_count,
-               "outage_count": outage_count})
-
-
-class Monitor:
-
-    def __init__(self):
-        self._result = []
-        self._monitor_process = []
-
-    def setup(self, config):
-        self._config = config
-
-    def start(self):
-        self._queue = multiprocessing.Queue()
-        self._event = multiprocessing.Event()
-        self._monitor_process = multiprocessing.Process(
-            target=_monitor_process, name="Monitor",
-            args=(self._config, self._queue, self._event))
-
-        self._monitor_process.start()
-        ret = self._queue.get()
-        if ret == "started":
-            LOG.debug("monitor process started!")
-
-    def stop(self):
-        self._event.set()
-        self._result = self._queue.get()
-        LOG.debug("stop the monitor process. the result:%s" % self._result)
-
-    def get_result(self):
-        return self._result
diff --git a/yardstick/benchmark/scenarios/availability/monitor/__init__.py b/yardstick/benchmark/scenarios/availability/monitor/__init__.py
new file mode 100755
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/__init__.py
diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
new file mode 100644
index 000000000..25990ac8c
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
@@ -0,0 +1,140 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import pkg_resources
+import logging
+import multiprocessing
+import time
+import os
+import yardstick.common.utils as utils
+
+LOG = logging.getLogger(__name__)
+
+monitor_conf_path = pkg_resources.resource_filename(
+    "yardstick.benchmark.scenarios.availability.monitor",
+    "monitor_conf.yaml")
+
+
+class BaseMonitor(multiprocessing.Process):
+    """docstring for BaseMonitor"""
+
+    def __init__(self, config, context):
+        multiprocessing.Process.__init__(self)
+        self._config = config
+        self._context = context
+        self._queue = multiprocessing.Queue()
+        self._event = multiprocessing.Event()
+        self.setup_done = False
+
+    @staticmethod
+    def get_monitor_cls(monitor_type):
+        '''return monitor class of specified type'''
+
+        for monitor in utils.itersubclasses(BaseMonitor):
+            if monitor_type == monitor.__monitor_type__:
+                return monitor
+        raise RuntimeError("No such monitor_type %s" % monitor_type)
+
+    def get_script_fullpath(self, path):
+        base_path = os.path.dirname(monitor_conf_path)
+        return os.path.join(base_path, path)
+
+    def run(self):
+        LOG.debug("config:%s context:%s" % (self._config, self._context))
+
+        self.setup()
+        monitor_time = self._config.get("monitor_time", 0)
+
+        total_time = 0
+        outage_time = 0
+        total_count = 0
+        outage_count = 0
+        first_outage = 0
+        last_outage = 0
+
+        begin_time = time.time()
+        while True:
+            total_count = total_count + 1
+
+            one_check_begin_time = time.time()
+            exit_status = self.monitor_func()
+            one_check_end_time = time.time()
+
+            if exit_status is False:
+                outage_count = outage_count + 1
+
+                outage_time = outage_time + (
+                    one_check_end_time - one_check_begin_time)
+
+                if not first_outage:
+                    first_outage = one_check_begin_time
+
+                last_outage = one_check_end_time
+
+            if self._event.is_set():
+                LOG.debug("the monitor process stop")
+                break
+
+            if one_check_end_time - begin_time > monitor_time:
+                LOG.debug("the monitor max_time finished and exit!")
+                break
+
+        end_time = time.time()
+        total_time = end_time - begin_time
+
+        self._queue.put({"total_time": total_time,
+                         "outage_time": last_outage-first_outage,
+                         "total_count": total_count,
+                         "outage_count": outage_count})
+
+    def start_monitor(self):
+        self.start()
+
+    def wait_monitor(self):
+        self.join()
+        self._result = self._queue.get()
+        LOG.debug("the monitor result:%s" % self._result)
+
+    def setup(self):  # pragma: no cover
+        pass
+
+    def monitor_func(self):  # pragma: no cover
+        pass
+
+    def verify_SLA(self):
+        pass
+
+
+class MonitorMgr(object):
+    """docstring for MonitorMgr"""
+    def __init__(self):
+        self._monitor_list = []
+
+    def init_monitors(self, monitor_cfgs, context):
+        LOG.debug("monitorMgr config: %s" % monitor_cfgs)
+
+        for monitor_cfg in monitor_cfgs:
+            monitor_type = monitor_cfg["monitor_type"]
+            monitor_cls = BaseMonitor.get_monitor_cls(monitor_type)
+            monitor_ins = monitor_cls(monitor_cfg, context)
+
+            self._monitor_list.append(monitor_ins)
+
+    def start_monitors(self):
+        for _monotor_instace in self._monitor_list:
+            _monotor_instace.start_monitor()
+
+    def wait_monitors(self):
+        for monitor in self._monitor_list:
+            monitor.wait_monitor()
+
+    def verify_SLA(self):
+        sla_pass = True
+        for monitor in self._monitor_list:
+            sla_pass = sla_pass & monitor.verify_SLA()
+        return sla_pass
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
new file mode 100644
index 000000000..232340aca
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
@@ -0,0 +1,81 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import logging
+import subprocess
+import traceback
+
+import basemonitor as basemonitor
+
+LOG = logging.getLogger(__name__)
+
+
+def _execute_shell_command(command):
+    '''execute shell script with error handling'''
+    exitcode = 0
+    output = []
+    try:
+        output = subprocess.check_output(command, shell=True)
+    except Exception:
+        exitcode = -1
+        output = traceback.format_exc()
+        LOG.error("exec command '%s' error:\n " % command)
+        LOG.error(traceback.format_exc())
+
+    return exitcode, output
+
+
+class MonitorOpenstackCmd(basemonitor.BaseMonitor):
+    """docstring for MonitorApi"""
+
+    __monitor_type__ = "openstack-cmd"
+
+    def monitor_func(self):
+        cmd = self._config["command_name"]
+        exit_status, stdout = _execute_shell_command(cmd)
+        if exit_status:
+            return False
+        return True
+
+    def verify_SLA(self):
+        outage_time = self._result.get('outage_time', None)
+        LOG.debug("the _result:%s" % self._result)
+        max_outage_time = self._config["sla"]["max_outage_time"]
+        if outage_time > max_outage_time:
+            LOG.info("SLA failure: %f > %f" % (outage_time, max_outage_time))
+            return False
+        else:
+            LOG.info("the sla is passed")
+            return True
+
+
+def _test():    # pragma: no cover
+    host = {
+        "ip": "10.20.0.5",
+        "user": "root",
+        "key_filename": "/root/.ssh/id_rsa"
+    }
+    context = {"node1": host}
+    monitor_configs = []
+    config = {
+        'monitor_type': 'openstack-cmd',
+        'command_name': 'nova image-list',
+        'monitor_time': 1,
+        'SLA': {'max_outage_time': 5}
+    }
+    monitor_configs.append(config)
+
+    p = basemonitor.MonitorMgr()
+    p.init_monitors(monitor_configs, context)
+    p.start_monitors()
+    p.wait_monitors()
+    p.verify()
+
+
+if __name__ == '__main__':    # pragma: no cover
+    _test()
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml b/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml
new file mode 100644
index 000000000..44f06038b
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml
@@ -0,0 +1,9 @@
+---
+# sample config file for ha test
+#
+schema: "yardstick:task:0.1"
+
+kill-process:
+ inject_script: scripts/stop_service.bash
+ recovery_script: scripts/start_service.bash
+ check_script: scripts/check_service.bash
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py
new file mode 100644
index 000000000..64e12f1dd
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py
@@ -0,0 +1,84 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import logging
+import yardstick.ssh as ssh
+
+import basemonitor as basemonitor
+
+LOG = logging.getLogger(__name__)
+
+
+class MonitorProcess(basemonitor.BaseMonitor):
+    """docstring for MonitorApi"""
+
+    __monitor_type__ = "process"
+
+    def setup(self):
+        host = self._context[self._config["host"]]
+        ip = host.get("ip", None)
+        user = host.get("user", "root")
+        key_filename = host.get("key_filename", "~/.ssh/id_rsa")
+
+        self.connection = ssh.SSH(user, ip, key_filename=key_filename)
+        self.connection.wait(timeout=600)
+        LOG.debug("ssh host success!")
+        self.check_script = self.get_script_fullpath(
+            "script_tools/check_service.bash")
+        self.process_name = self._config["process_name"]
+
+    def monitor_func(self):
+        exit_status, stdout, stderr = self.connection.execute(
+            "/bin/sh -s {0}".format(self.process_name),
+            stdin=open(self.check_script, "r"))
+
+        if stdout and "running" in stdout:
+            LOG.info("check the envrioment success!")
+            return True
+        else:
+            LOG.error(
+                "the host envrioment is error, stdout:%s, stderr:%s" %
+                (stdout, stderr))
+        return False
+
+    def verify_SLA(self):
+        outage_time = self._result.get('outage_time', None)
+        max_outage_time = self._config["sla"]["max_recover_time"]
+        if outage_time > max_outage_time:
+            LOG.error("SLA failure: %f > %f" % (outage_time, max_outage_time))
+            return False
+        else:
+            return True
+
+
+def _test():    # pragma: no cover
+    host = {
+        "ip": "10.20.0.5",
+        "user": "root",
+        "key_filename": "/root/.ssh/id_rsa"
+    }
+    context = {"node1": host}
+    monitor_configs = []
+    config = {
+        'monitor_type': 'process',
+        'process_name': 'nova-api',
+        'host': "node1",
+        'monitor_time': 1,
+        'SLA': {'max_recover_time': 5}
+    }
+    monitor_configs.append(config)
+
+    p = basemonitor.MonitorMgr()
+    p.init_monitors(monitor_configs, context)
+    p.start_monitors()
+    p.wait_monitors()
+    p.verify()
+
+
+if __name__ == '__main__':    # pragma: no cover
+    _test()
diff --git a/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash b/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash
new file mode 100755
index 000000000..cc898a859
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# check the status of a service
+
+set -e
+
+service_name=$1
+
+service $service_name status
diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py
index 10134ea6d..aee94ee09 100755
--- a/yardstick/benchmark/scenarios/availability/serviceha.py
+++ b/yardstick/benchmark/scenarios/availability/serviceha.py
@@ -7,9 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 import logging
-import time
 from yardstick.benchmark.scenarios import base
-from yardstick.benchmark.scenarios.availability import monitor
+from yardstick.benchmark.scenarios.availability.monitor import basemonitor
 from yardstick.benchmark.scenarios.availability.attacker import baseattacker
 
 LOG = logging.getLogger(__name__)
@@ -34,6 +33,7 @@ class ServiceHA(base.Scenario):
         if nodes is None:
             LOG.error("the nodes info is none")
             return
+
         self.attackers = []
         attacker_cfgs = self.scenario_cfg["options"]["attackers"]
         for attacker_cfg in attacker_cfgs:
@@ -45,9 +45,9 @@ class ServiceHA(base.Scenario):
 
         monitor_cfgs = self.scenario_cfg["options"]["monitors"]
 
-        self.monitor_ins = monitor.Monitor()
-        self.monitor_ins.setup(monitor_cfgs[0])
-        self.monitor_ins.monitor_time = monitor_cfgs[0]["monitor_time"]
+        self.monitorMgr = basemonitor.MonitorMgr()
+        self.monitorMgr.init_monitors(monitor_cfgs, nodes)
+
         self.setup_done = True
 
     def run(self, result):
@@ -56,35 +56,24 @@ class ServiceHA(base.Scenario):
             LOG.error("The setup not finished!")
             return
 
-        self.monitor_ins.start()
+        self.monitorMgr.start_monitors()
         LOG.info("monitor start!")
 
         for attacker in self.attackers:
             attacker.inject_fault()
 
-        time.sleep(self.monitor_ins.monitor_time)
-
-        self.monitor_ins.stop()
+        self.monitorMgr.wait_monitors()
         LOG.info("monitor stop!")
 
-        ret = self.monitor_ins.get_result()
-        LOG.info("The monitor result:%s" % ret)
-        outage_time = ret.get("outage_time")
-        result["outage_time"] = outage_time
-        LOG.info("the result:%s" % result)
-
-        if "sla" in self.scenario_cfg:
-            sla_outage_time = int(self.scenario_cfg["sla"]["outage_time"])
-            assert outage_time <= sla_outage_time, "outage_time %f > sla:outage_time(%f)" % \
-                (outage_time, sla_outage_time)
+        sla_pass = self.monitorMgr.verify_SLA()
+        assert sla_pass is True, "the test cases is not pass the SLA"
 
         return
 
     def teardown(self):
         '''scenario teardown'''
         for attacker in self.attackers:
-            if not attacker.check():
-                attacker.recover()
+            attacker.recover()
 
 
 def _test():    # pragma: no cover
@@ -103,14 +92,14 @@ def _test():    # pragma: no cover
     attacker_cfgs = []
     attacker_cfgs.append(attacker_cfg)
     monitor_cfg = {
-        "monitor_cmd": "nova image-list",
-        "monitor_tme": 10
+        "monitor_cmd": "nova image-list"
     }
     monitor_cfgs = []
     monitor_cfgs.append(monitor_cfg)
 
     options = {
         "attackers": attacker_cfgs,
+        "wait_time": 10,
         "monitors": monitor_cfgs
     }
     sla = {"outage_time": 5}