Rewrite the HA test case (2)

idea: refact the Monitor class in old file "monitor.py" with the base
class and sub-class.
detail:
1) the BaseMonitor is the base class of other monitor
2) each monitor run in independent process
3) there are two monitor("openstack-cmd" and "process") for the first test case
4) MonitorMgr class used to manager monitor process

JIRA: YARDSTICK-149

Change-Id: I2eede94481f740812212e6cb673d175b5f543c15
Signed-off-by: wym_libra <yimin.wang@huawei.com>
(cherry picked from commit 2733defda816a84d2b9c2e361a5970b9de4923f4)

8 files changed, 344 insertions, 137 deletions

diff --git a/yardstick/benchmark/scenarios/availability/monitor.py b/yardstick/benchmark/scenarios/availability/monitor.py
deleted file mode 100755
index 3193d3304..000000000
--- a/yardstick/benchmark/scenarios/availability/monitor.py
+++ /dev/null
@@ -1,114 +0,0 @@
-##############################################################################
-# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-import logging
-import multiprocessing
-import subprocess
-import traceback
-import time
-
-LOG = logging.getLogger(__name__)
-
-
-def _execute_shell_command(command):
-    '''execute shell script with error handling'''
-    exitcode = 0
-    output = []
-    try:
-        output = subprocess.check_output(command, shell=True)
-    except Exception:
-        exitcode = -1
-        output = traceback.format_exc()
-        LOG.error("exec command '%s' error:\n " % command)
-        LOG.error(traceback.format_exc())
-
-    return exitcode, output
-
-
-def _monitor_process(config, queue, event):
-
-    total_time = 0
-    outage_time = 0
-    total_count = 0
-    outage_count = 0
-    first_outage = 0
-    last_outage = 0
-
-    wait_time = config.get("duration", 0)
-    cmd = config.get("monitor_cmd", None)
-    if cmd is None:
-        LOG.error("There are no monitor cmd!")
-        return
-
-    queue.put("started")
-
-    begin_time = time.time()
-    while True:
-
-        total_count = total_count + 1
-
-        one_check_begin_time = time.time()
-        exit_status, stdout = _execute_shell_command(cmd)
-        one_check_end_time = time.time()
-
-        LOG.info("the exit_status:%s stdout:%s" % (exit_status, stdout))
-        if exit_status:
-            outage_count = outage_count + 1
-
-            outage_time = outage_time + (
-                one_check_end_time - one_check_begin_time)
-
-            if not first_outage:
-                first_outage = one_check_begin_time
-
-            last_outage = one_check_end_time
-
-        if event.is_set():
-            LOG.debug("the monitor process stop")
-            break
-
-        if wait_time > 0:
-            time.sleep(wait_time)
-
-    end_time = time.time()
-    total_time = end_time - begin_time
-
-    queue.put({"total_time": total_time,
-               "outage_time": last_outage-first_outage,
-               "total_count": total_count,
-               "outage_count": outage_count})
-
-
-class Monitor:
-
-    def __init__(self):
-        self._result = []
-        self._monitor_process = []
-
-    def setup(self, config):
-        self._config = config
-
-    def start(self):
-        self._queue = multiprocessing.Queue()
-        self._event = multiprocessing.Event()
-        self._monitor_process = multiprocessing.Process(
-            target=_monitor_process, name="Monitor",
-            args=(self._config, self._queue, self._event))
-
-        self._monitor_process.start()
-        ret = self._queue.get()
-        if ret == "started":
-            LOG.debug("monitor process started!")
-
-    def stop(self):
-        self._event.set()
-        self._result = self._queue.get()
-        LOG.debug("stop the monitor process. the result:%s" % self._result)
-
-    def get_result(self):
-        return self._result
diff --git a/yardstick/benchmark/scenarios/availability/monitor/__init__.py b/yardstick/benchmark/scenarios/availability/monitor/__init__.py
new file mode 100755
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/__init__.py
diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
new file mode 100644
index 000000000..25990ac8c
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
@@ -0,0 +1,140 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import pkg_resources
+import logging
+import multiprocessing
+import time
+import os
+import yardstick.common.utils as utils
+
+LOG = logging.getLogger(__name__)
+
+monitor_conf_path = pkg_resources.resource_filename(
+    "yardstick.benchmark.scenarios.availability.monitor",
+    "monitor_conf.yaml")
+
+
+class BaseMonitor(multiprocessing.Process):
+    """docstring for BaseMonitor"""
+
+    def __init__(self, config, context):
+        multiprocessing.Process.__init__(self)
+        self._config = config
+        self._context = context
+        self._queue = multiprocessing.Queue()
+        self._event = multiprocessing.Event()
+        self.setup_done = False
+
+    @staticmethod
+    def get_monitor_cls(monitor_type):
+        '''return monitor class of specified type'''
+
+        for monitor in utils.itersubclasses(BaseMonitor):
+            if monitor_type == monitor.__monitor_type__:
+                return monitor
+        raise RuntimeError("No such monitor_type %s" % monitor_type)
+
+    def get_script_fullpath(self, path):
+        base_path = os.path.dirname(monitor_conf_path)
+        return os.path.join(base_path, path)
+
+    def run(self):
+        LOG.debug("config:%s context:%s" % (self._config, self._context))
+
+        self.setup()
+        monitor_time = self._config.get("monitor_time", 0)
+
+        total_time = 0
+        outage_time = 0
+        total_count = 0
+        outage_count = 0
+        first_outage = 0
+        last_outage = 0
+
+        begin_time = time.time()
+        while True:
+            total_count = total_count + 1
+
+            one_check_begin_time = time.time()
+            exit_status = self.monitor_func()
+            one_check_end_time = time.time()
+
+            if exit_status is False:
+                outage_count = outage_count + 1
+
+                outage_time = outage_time + (
+                    one_check_end_time - one_check_begin_time)
+
+                if not first_outage:
+                    first_outage = one_check_begin_time
+
+                last_outage = one_check_end_time
+
+            if self._event.is_set():
+                LOG.debug("the monitor process stop")
+                break
+
+            if one_check_end_time - begin_time > monitor_time:
+                LOG.debug("the monitor max_time finished and exit!")
+                break
+
+        end_time = time.time()
+        total_time = end_time - begin_time
+
+        self._queue.put({"total_time": total_time,
+                         "outage_time": last_outage-first_outage,
+                         "total_count": total_count,
+                         "outage_count": outage_count})
+
+    def start_monitor(self):
+        self.start()
+
+    def wait_monitor(self):
+        self.join()
+        self._result = self._queue.get()
+        LOG.debug("the monitor result:%s" % self._result)
+
+    def setup(self):  # pragma: no cover
+        pass
+
+    def monitor_func(self):  # pragma: no cover
+        pass
+
+    def verify_SLA(self):
+        pass
+
+
+class MonitorMgr(object):
+    """docstring for MonitorMgr"""
+    def __init__(self):
+        self._monitor_list = []
+
+    def init_monitors(self, monitor_cfgs, context):
+        LOG.debug("monitorMgr config: %s" % monitor_cfgs)
+
+        for monitor_cfg in monitor_cfgs:
+            monitor_type = monitor_cfg["monitor_type"]
+            monitor_cls = BaseMonitor.get_monitor_cls(monitor_type)
+            monitor_ins = monitor_cls(monitor_cfg, context)
+
+            self._monitor_list.append(monitor_ins)
+
+    def start_monitors(self):
+        for _monotor_instace in self._monitor_list:
+            _monotor_instace.start_monitor()
+
+    def wait_monitors(self):
+        for monitor in self._monitor_list:
+            monitor.wait_monitor()
+
+    def verify_SLA(self):
+        sla_pass = True
+        for monitor in self._monitor_list:
+            sla_pass = sla_pass & monitor.verify_SLA()
+        return sla_pass
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
new file mode 100644
index 000000000..232340aca
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
@@ -0,0 +1,81 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import logging
+import subprocess
+import traceback
+
+import basemonitor as basemonitor
+
+LOG = logging.getLogger(__name__)
+
+
+def _execute_shell_command(command):
+    '''execute shell script with error handling'''
+    exitcode = 0
+    output = []
+    try:
+        output = subprocess.check_output(command, shell=True)
+    except Exception:
+        exitcode = -1
+        output = traceback.format_exc()
+        LOG.error("exec command '%s' error:\n " % command)
+        LOG.error(traceback.format_exc())
+
+    return exitcode, output
+
+
+class MonitorOpenstackCmd(basemonitor.BaseMonitor):
+    """docstring for MonitorApi"""
+
+    __monitor_type__ = "openstack-cmd"
+
+    def monitor_func(self):
+        cmd = self._config["command_name"]
+        exit_status, stdout = _execute_shell_command(cmd)
+        if exit_status:
+            return False
+        return True
+
+    def verify_SLA(self):
+        outage_time = self._result.get('outage_time', None)
+        LOG.debug("the _result:%s" % self._result)
+        max_outage_time = self._config["sla"]["max_outage_time"]
+        if outage_time > max_outage_time:
+            LOG.info("SLA failure: %f > %f" % (outage_time, max_outage_time))
+            return False
+        else:
+            LOG.info("the sla is passed")
+            return True
+
+
+def _test():    # pragma: no cover
+    host = {
+        "ip": "10.20.0.5",
+        "user": "root",
+        "key_filename": "/root/.ssh/id_rsa"
+    }
+    context = {"node1": host}
+    monitor_configs = []
+    config = {
+        'monitor_type': 'openstack-cmd',
+        'command_name': 'nova image-list',
+        'monitor_time': 1,
+        'SLA': {'max_outage_time': 5}
+    }
+    monitor_configs.append(config)
+
+    p = basemonitor.MonitorMgr()
+    p.init_monitors(monitor_configs, context)
+    p.start_monitors()
+    p.wait_monitors()
+    p.verify()
+
+
+if __name__ == '__main__':    # pragma: no cover
+    _test()
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml b/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml
new file mode 100644
index 000000000..44f06038b
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_conf.yaml
@@ -0,0 +1,9 @@
+---
+# sample config file for ha test
+#
+schema: "yardstick:task:0.1"
+
+kill-process:
+ inject_script: scripts/stop_service.bash
+ recovery_script: scripts/start_service.bash
+ check_script: scripts/check_service.bash
diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py
new file mode 100644
index 000000000..64e12f1dd
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_process.py
@@ -0,0 +1,84 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import logging
+import yardstick.ssh as ssh
+
+import basemonitor as basemonitor
+
+LOG = logging.getLogger(__name__)
+
+
+class MonitorProcess(basemonitor.BaseMonitor):
+    """docstring for MonitorApi"""
+
+    __monitor_type__ = "process"
+
+    def setup(self):
+        host = self._context[self._config["host"]]
+        ip = host.get("ip", None)
+        user = host.get("user", "root")
+        key_filename = host.get("key_filename", "~/.ssh/id_rsa")
+
+        self.connection = ssh.SSH(user, ip, key_filename=key_filename)
+        self.connection.wait(timeout=600)
+        LOG.debug("ssh host success!")
+        self.check_script = self.get_script_fullpath(
+            "script_tools/check_service.bash")
+        self.process_name = self._config["process_name"]
+
+    def monitor_func(self):
+        exit_status, stdout, stderr = self.connection.execute(
+            "/bin/sh -s {0}".format(self.process_name),
+            stdin=open(self.check_script, "r"))
+
+        if stdout and "running" in stdout:
+            LOG.info("check the envrioment success!")
+            return True
+        else:
+            LOG.error(
+                "the host envrioment is error, stdout:%s, stderr:%s" %
+                (stdout, stderr))
+        return False
+
+    def verify_SLA(self):
+        outage_time = self._result.get('outage_time', None)
+        max_outage_time = self._config["sla"]["max_recover_time"]
+        if outage_time > max_outage_time:
+            LOG.error("SLA failure: %f > %f" % (outage_time, max_outage_time))
+            return False
+        else:
+            return True
+
+
+def _test():    # pragma: no cover
+    host = {
+        "ip": "10.20.0.5",
+        "user": "root",
+        "key_filename": "/root/.ssh/id_rsa"
+    }
+    context = {"node1": host}
+    monitor_configs = []
+    config = {
+        'monitor_type': 'process',
+        'process_name': 'nova-api',
+        'host': "node1",
+        'monitor_time': 1,
+        'SLA': {'max_recover_time': 5}
+    }
+    monitor_configs.append(config)
+
+    p = basemonitor.MonitorMgr()
+    p.init_monitors(monitor_configs, context)
+    p.start_monitors()
+    p.wait_monitors()
+    p.verify()
+
+
+if __name__ == '__main__':    # pragma: no cover
+    _test()
diff --git a/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash b/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash
new file mode 100755
index 000000000..cc898a859
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor/script_tools/check_service.bash
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# check the status of a service
+
+set -e
+
+service_name=$1
+
+service $service_name status
diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py
index 10134ea6d..aee94ee09 100755
--- a/yardstick/benchmark/scenarios/availability/serviceha.py
+++ b/yardstick/benchmark/scenarios/availability/serviceha.py
@@ -7,9 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 import logging
-import time
 from yardstick.benchmark.scenarios import base
-from yardstick.benchmark.scenarios.availability import monitor
+from yardstick.benchmark.scenarios.availability.monitor import basemonitor
 from yardstick.benchmark.scenarios.availability.attacker import baseattacker
 
 LOG = logging.getLogger(__name__)
@@ -34,6 +33,7 @@ class ServiceHA(base.Scenario):
         if nodes is None:
             LOG.error("the nodes info is none")
             return
+
         self.attackers = []
         attacker_cfgs = self.scenario_cfg["options"]["attackers"]
         for attacker_cfg in attacker_cfgs:
@@ -45,9 +45,9 @@ class ServiceHA(base.Scenario):
 
         monitor_cfgs = self.scenario_cfg["options"]["monitors"]
 
-        self.monitor_ins = monitor.Monitor()
-        self.monitor_ins.setup(monitor_cfgs[0])
-        self.monitor_ins.monitor_time = monitor_cfgs[0]["monitor_time"]
+        self.monitorMgr = basemonitor.MonitorMgr()
+        self.monitorMgr.init_monitors(monitor_cfgs, nodes)
+
         self.setup_done = True
 
     def run(self, result):
@@ -56,35 +56,24 @@ class ServiceHA(base.Scenario):
             LOG.error("The setup not finished!")
             return
 
-        self.monitor_ins.start()
+        self.monitorMgr.start_monitors()
         LOG.info("monitor start!")
 
         for attacker in self.attackers:
             attacker.inject_fault()
 
-        time.sleep(self.monitor_ins.monitor_time)
-
-        self.monitor_ins.stop()
+        self.monitorMgr.wait_monitors()
         LOG.info("monitor stop!")
 
-        ret = self.monitor_ins.get_result()
-        LOG.info("The monitor result:%s" % ret)
-        outage_time = ret.get("outage_time")
-        result["outage_time"] = outage_time
-        LOG.info("the result:%s" % result)
-
-        if "sla" in self.scenario_cfg:
-            sla_outage_time = int(self.scenario_cfg["sla"]["outage_time"])
-            assert outage_time <= sla_outage_time, "outage_time %f > sla:outage_time(%f)" % \
-                (outage_time, sla_outage_time)
+        sla_pass = self.monitorMgr.verify_SLA()
+        assert sla_pass is True, "the test cases is not pass the SLA"
 
         return
 
     def teardown(self):
         '''scenario teardown'''
         for attacker in self.attackers:
-            if not attacker.check():
-                attacker.recover()
+            attacker.recover()
 
 
 def _test():    # pragma: no cover
@@ -103,14 +92,14 @@ def _test():    # pragma: no cover
     attacker_cfgs = []
     attacker_cfgs.append(attacker_cfg)
     monitor_cfg = {
-        "monitor_cmd": "nova image-list",
-        "monitor_tme": 10
+        "monitor_cmd": "nova image-list"
     }
     monitor_cfgs = []
     monitor_cfgs.append(monitor_cfg)
 
     options = {
         "attackers": attacker_cfgs,
+        "wait_time": 10,
         "monitors": monitor_cfgs
     }
     sla = {"outage_time": 5}