A initial HA test case

1)stop an openstack service
2)then monitor the corresponding api and check the availability of it
3)recovery the openstack service

JIRA: YARDSTICK-149

Change-Id: Id7b77d2f5c71844729c04f37442c8cfaa270ab12
Signed-off-by: wym_libra <yimin.wang@huawei.com>

7 files changed, 376 insertions, 0 deletions

diff --git a/yardstick/benchmark/scenarios/availability/__init__.py b/yardstick/benchmark/scenarios/availability/__init__.py
new file mode 100755
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/__init__.py
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/check_service.bash b/yardstick/benchmark/scenarios/availability/ha_tools/check_service.bash
new file mode 100755
index 000000000..cc898a859
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/check_service.bash
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# check the status of a service
+
+set -e
+
+service_name=$1
+
+service $service_name status
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/ha_conf.yaml b/yardstick/benchmark/scenarios/availability/ha_tools/ha_conf.yaml
new file mode 100644
index 000000000..67e56eb4f
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/ha_conf.yaml
@@ -0,0 +1,12 @@
+---
+# sample config file for ha test
+#
+schema: "yardstick:task:0.1"
+
+nova-api:
+-
+ type: stop-service
+ inject_script: ha_tools/stop_service.bash
+ recovery_script: ha_tools/start_service.bash
+ check_script: ha_tools/check_service.bash
+ monitor_cmd: nova image-list
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/start_service.bash b/yardstick/benchmark/scenarios/availability/ha_tools/start_service.bash
new file mode 100755
index 000000000..c1bf8b7eb
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/start_service.bash
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Start a service and check the service is started
+
+set -e
+
+service_name=$1
+
+service $service_name start
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/stop_service.bash b/yardstick/benchmark/scenarios/availability/ha_tools/stop_service.bash
new file mode 100755
index 000000000..a8901784e
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/stop_service.bash
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Stop a service and check the service is stoped
+
+set -e
+
+service_name=$1
+
+service $service_name stop
+
+# TODO
+# check the service status
diff --git a/yardstick/benchmark/scenarios/availability/monitor.py b/yardstick/benchmark/scenarios/availability/monitor.py
new file mode 100755
index 000000000..3193d3304
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/monitor.py
@@ -0,0 +1,114 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import logging
+import multiprocessing
+import subprocess
+import traceback
+import time
+
+LOG = logging.getLogger(__name__)
+
+
+def _execute_shell_command(command):
+    '''execute shell script with error handling'''
+    exitcode = 0
+    output = []
+    try:
+        output = subprocess.check_output(command, shell=True)
+    except Exception:
+        exitcode = -1
+        output = traceback.format_exc()
+        LOG.error("exec command '%s' error:\n " % command)
+        LOG.error(traceback.format_exc())
+
+    return exitcode, output
+
+
+def _monitor_process(config, queue, event):
+
+    total_time = 0
+    outage_time = 0
+    total_count = 0
+    outage_count = 0
+    first_outage = 0
+    last_outage = 0
+
+    wait_time = config.get("duration", 0)
+    cmd = config.get("monitor_cmd", None)
+    if cmd is None:
+        LOG.error("There are no monitor cmd!")
+        return
+
+    queue.put("started")
+
+    begin_time = time.time()
+    while True:
+
+        total_count = total_count + 1
+
+        one_check_begin_time = time.time()
+        exit_status, stdout = _execute_shell_command(cmd)
+        one_check_end_time = time.time()
+
+        LOG.info("the exit_status:%s stdout:%s" % (exit_status, stdout))
+        if exit_status:
+            outage_count = outage_count + 1
+
+            outage_time = outage_time + (
+                one_check_end_time - one_check_begin_time)
+
+            if not first_outage:
+                first_outage = one_check_begin_time
+
+            last_outage = one_check_end_time
+
+        if event.is_set():
+            LOG.debug("the monitor process stop")
+            break
+
+        if wait_time > 0:
+            time.sleep(wait_time)
+
+    end_time = time.time()
+    total_time = end_time - begin_time
+
+    queue.put({"total_time": total_time,
+               "outage_time": last_outage-first_outage,
+               "total_count": total_count,
+               "outage_count": outage_count})
+
+
+class Monitor:
+
+    def __init__(self):
+        self._result = []
+        self._monitor_process = []
+
+    def setup(self, config):
+        self._config = config
+
+    def start(self):
+        self._queue = multiprocessing.Queue()
+        self._event = multiprocessing.Event()
+        self._monitor_process = multiprocessing.Process(
+            target=_monitor_process, name="Monitor",
+            args=(self._config, self._queue, self._event))
+
+        self._monitor_process.start()
+        ret = self._queue.get()
+        if ret == "started":
+            LOG.debug("monitor process started!")
+
+    def stop(self):
+        self._event.set()
+        self._result = self._queue.get()
+        LOG.debug("stop the monitor process. the result:%s" % self._result)
+
+    def get_result(self):
+        return self._result
diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py
new file mode 100755
index 000000000..3e03e1da5
--- /dev/null
+++ b/yardstick/benchmark/scenarios/availability/serviceha.py
@@ -0,0 +1,193 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import pkg_resources
+import logging
+import time
+import yaml
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+from yardstick.benchmark.scenarios.availability import monitor
+
+LOG = logging.getLogger(__name__)
+
+
+class ServiceHA(base.Scenario):
+    """TODO: docstring of ServiceHA
+    """
+    __scenario_type__ = "ServiceHA"
+
+    HA_CONF = "ha_tools/ha_conf.yaml"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.service_name = scenario_cfg["options"]["component"]
+        self.fault_type = scenario_cfg["options"]["fault_type"]
+        self.fault_time = scenario_cfg["options"].get("fault_time", 0)
+        self.fault_cfg = None
+        self.setup_done = False
+        self.need_teardown = False
+
+    def setup(self):
+        '''scenario setup'''
+        self.ha_conf_file = pkg_resources.resource_filename(
+            "yardstick.benchmark.scenarios.availability",
+            ServiceHA.HA_CONF)
+        ha_cfg = []
+        with open(self.ha_conf_file) as stream:
+                ha_cfg = yaml.load(stream)
+        LOG.debug("ha_cfg content:%s" % ha_cfg)
+
+        # check the ha_conf contains the service defined in test cases yaml
+        service_cfg = ha_cfg.get(self.service_name, None)
+        if not service_cfg:
+            LOG.error(
+                "The component %s can not be supported!" % self.service_name)
+            return
+
+        for fault in service_cfg:
+            if fault["type"] == self.fault_type:
+                self.fault_cfg = fault
+                break
+        if not self.fault_cfg:
+            LOG.error(
+                "The fualt_type %s can not be supproted!" % self.fault_type)
+            return
+        LOG.debug("the fault_cfg :%s" % self.fault_cfg)
+
+        self.fault_script = pkg_resources.resource_filename(
+            "yardstick.benchmark.scenarios.availability",
+            self.fault_cfg["inject_script"])
+        self.recovery_script = pkg_resources.resource_filename(
+            "yardstick.benchmark.scenarios.availability",
+            self.fault_cfg["recovery_script"])
+        self.check_script = pkg_resources.resource_filename(
+            "yardstick.benchmark.scenarios.availability",
+            self.fault_cfg["check_script"])
+
+        host = self.context_cfg.get("host", None)
+        ip = host.get("ip", None)
+        user = host.get("user", "root")
+        key_filename = host.get("key_filename", "~/.ssh/id_rsa")
+        LOG.info("The host: %s  the service: %s" % (ip, self.service_name))
+        LOG.debug("The params, host:%s  fault_cfg:%s" % (host, self.fault_cfg))
+
+        LOG.debug(
+            "ssh connection ip:%s, user:%s, key_file:%s",
+            ip, user, key_filename)
+        self.connection = ssh.SSH(user, ip, key_filename=key_filename)
+        self.connection.wait(timeout=600)
+        LOG.debug("ssh host success!")
+
+        # check the host envrioment
+        exit_status, stdout, stderr = self.connection.execute(
+            "/bin/sh -s {0}".format(self.service_name),
+            stdin=open(self.check_script, "r"))
+        LOG.info(
+            "the exit_status:%s stdout:%s stderr:%s" %
+            (exit_status, stdout, stderr))
+        if exit_status:
+            raise RuntimeError(stderr)
+
+        if stdout and "running" in stdout:
+            LOG.info("check the envrioment success!")
+        else:
+            LOG.error(
+                "the host envrioment is error, stdout:%s, stderr:%s" %
+                (stdout, stderr))
+            return
+
+        self.setup_done = True
+
+    def run(self, result):
+        """execute the benchmark"""
+        if not self.setup_done:
+            LOG.error("The setup not finished!")
+            return
+
+        monitorInstance = monitor.Monitor()
+        monitorInstance.setup(self.fault_cfg)
+        monitorInstance.start()
+        LOG.info("monitor start!")
+
+        LOG.info("Inject fault!")
+        exit_status, stdout, stderr = self.connection.execute(
+            "/bin/sh -s {0}".format(self.service_name),
+            stdin=open(self.fault_script, "r"))
+
+        if exit_status != 0:
+            monitorInstance.stop()
+            raise RuntimeError(stderr)
+
+        self.need_teardown = True
+        time.sleep(self.fault_time)
+
+        monitorInstance.stop()
+        LOG.info("monitor stop!")
+
+        ret = monitorInstance.get_result()
+        LOG.info("The monitor result:%s" % ret)
+        outage_time = ret.get("outage_time")
+        result["outage_time"] = outage_time
+        LOG.info("the result:%s" % result)
+
+        if "sla" in self.scenario_cfg:
+            sla_outage_time = int(self.scenario_cfg["sla"]["outage_time"])
+            assert outage_time <= sla_outage_time, "outage_time %f > sla:outage_time(%f)" % \
+                (outage_time, sla_outage_time)
+
+        return
+
+    def teardown(self):
+        '''scenario teardown'''
+        LOG.info("recory the everiment!")
+
+        if self.need_teardown:
+            exit_status, stdout, stderr = self.connection.execute(
+                "/bin/sh -s {0} ".format(self.service_name),
+                stdin=open(self.recovery_script, "r"))
+
+            if exit_status:
+                raise RuntimeError(stderr)
+            else:
+                self.need_teardown = False
+
+"""
+def _test():
+    '''internal test function'''
+    host = {
+        "ip": "10.20.0.5",
+        "user": "root",
+        "key_filename": "/root/.ssh/id_rsa"
+    }
+    ctx = {"host": host}
+
+    logger = logging.getLogger("yardstick")
+    logger.setLevel(logging.DEBUG)
+
+    options = {
+        "component": "nova-api",
+        "fault_type": "stop-service"
+    }
+    sla = {"outage_time": 5}
+    args = {"options": options, "sla": sla}
+
+    print "create instance"
+    terstInstance = ServiceHA(args, ctx)
+
+    terstInstance.setup()
+    result = {}
+    terstInstance.run(result)
+    print result
+
+    terstInstance.teardown()
+
+if __name__ == '__main__':
+    _test()
+"""