Merge "Test case: Fio volume benchmark testcase using job file"

4 files changed, 280 insertions, 37 deletions

diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc006.rst b/docs/testing/user/userguide/opnfv_yardstick_tc006.rst
new file mode 100644
index 000000000..d2d6467f1
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc006.rst
@@ -0,0 +1,119 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC006
+*************************************
+
+.. _fio: http://bluestop.org/files/fio/HOWTO.txt
+
++-----------------------------------------------------------------------------+
+|Volume storage Performance                                                   |
+|                                                                             |
++--------------+--------------------------------------------------------------+
+|test case id  | OPNFV_YARDSTICK_TC006_VOLUME STORAGE PERFORMANCE             |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|metric        | IOPS (Average IOs performed per second),                     |
+|              | Throughput (Average disk read/write bandwidth rate),         |
+|              | Latency (Average disk read/write latency)                    |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test purpose  | The purpose of TC006 is to evaluate the IaaS volume storage  |
+|              | performance with regards to IOPS, throughput and latency.    |
+|              |                                                              |
+|              | The purpose is also to be able to spot the trends.           |
+|              | Test results, graphs and similar shall be stored for         |
+|              | comparison reasons and product evolution understanding       |
+|              | between different OPNFV versions and/or configurations.      |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test tool     | fio                                                          |
+|              |                                                              |
+|              | fio is an I/O tool meant to be used both for benchmark and   |
+|              | stress/hardware verification. It has support for 19          |
+|              | different types of I/O engines (sync, mmap, libaio,          |
+|              | posixaio, SG v3, splice, null, network, syslet, guasi,       |
+|              | solarisaio, and more), I/O priorities (for newer Linux       |
+|              | kernels), rate I/O, forked or threaded jobs, and much more.  |
+|              |                                                              |
+|              | (fio is not always part of a Linux distribution, hence it    |
+|              | needs to be installed. As an example see the                 |
+|              | /yardstick/tools/ directory for how to generate a Linux      |
+|              | image with fio included.)                                    |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test          | fio test is invoked in a host VM with a volume attached on a |
+|description   | compute blade, a job file as well as parameters are passed   |
+|              | to fio and fio will start doing what the job file tells it   |
+|              | to do.                                                       |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc006.yaml                             |
+|              |                                                              |
+|              | Fio job file is provided to define the benchmark process     |
+|              | Target volume is mounted at /FIO_Test directory              |
+|              |                                                              |
+|              | For SLA, minimum read/write iops is set to 100,              |
+|              | minimum read/write throughput is set to 400 KB/s,            |
+|              | and maximum read/write latency is set to 20000 usec.         |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|applicability | This test case can be configured with different:             |
+|              |                                                              |
+|              |   * Job file;                                                |
+|              |   * Volume mount directory.                                  |
+|              |                                                              |
+|              | SLA is optional. The SLA in this test case serves as an      |
+|              | example. Considerably higher throughput and lower latency    |
+|              | are expected. However, to cover most configurations, both    |
+|              | baremetal and fully virtualized  ones, this value should be  |
+|              | possible to achieve and acceptable for black box testing.    |
+|              | Many heavy IO applications start to suffer badly if the      |
+|              | read/write bandwidths are lower than this.                   |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|usability     | This test case is one of Yardstick's generic test. Thus it   |
+|              | is runnable on most of the scenarios.                        |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|references    | fio_                                                         |
+|              |                                                              |
+|              | ETSI-NFV-TST001                                              |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|pre-test      | The test case image needs to be installed into Glance        |
+|conditions    | with fio included in it.                                     |
+|              |                                                              |
+|              | No POD specific requirements have been identified.           |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result                              |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 1        | A host VM with fio installed is booted.                      |
+|              | A 200G volume is attached to the host VM                     |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 2        | Yardstick is connected with the host VM by using ssh.        |
+|              | 'job_file.ini' is copyied from Jump Host to the host VM via  |
+|              | the ssh tunnel. The attached volume is formated and mounted. |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 3        | Fio benchmark is invoked. Simulated IO operations are        |
+|              | started. IOPS, disk read/write bandwidth and latency are     |
+|              | recorded and checked against the SLA. Logs are produced and  |
+|              | stored.                                                      |
+|              |                                                              |
+|              | Result: Logs are stored.                                     |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 4        | The host VM is deleted.                                      |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test verdict  | Fails only if SLA is not passed, or if there is a test case  |
+|              | execution problem.                                           |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc006.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc006.yaml
new file mode 100644
index 000000000..a35629f81
--- /dev/null
+++ b/tests/opnfv/test_cases/opnfv_yardstick_tc006.yaml
@@ -0,0 +1,68 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+
+schema: "yardstick:task:0.1"
+description: >
+    Yardstick TC006 config file;
+    Measure volume storage IOPS, throughput and latency using fio with job file.
+
+{% set directory = directory or "/FIO_Test" %}
+{% set provider = provider or none %}
+{% set physical_network = physical_network or 'physnet1' %}
+{% set segmentation_id = segmentation_id or none %}
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+  type: Fio
+  options:
+    job_file: "job_file.ini"
+    directory: {{ directory }}
+
+  host: fio.yardstick-TC006
+
+  runner:
+    type: Iteration
+    iterations: 1
+    interval: 1
+
+  sla:
+    read_bw: 6000
+    read_iops: 1500
+    read_lat: 500.1
+    write_bw: 6000
+    write_iops: 1500
+    write_lat: 500.1
+    action: monitor
+
+context:
+  name: yardstick-TC006
+  image: yardstick-image
+  flavor: yardstick-flavor
+  user: ubuntu
+  servers:
+    fio:
+      floating_ip: true
+      volume:
+        name: test-volume
+        size: 200
+      volume_mountpoint: /dev/vdb
+
+  networks:
+    test:
+      cidr: '10.0.1.0/24'
+      {% if provider == "vlan" %}
+      provider: {{provider}}
+      physical_network: {{physical_network}}
+        {% if segmentation_id %}
+      segmentation_id: {{segmentation_id}}
+        {% endif %}
+      {% endif %}
diff --git a/tests/unit/benchmark/scenarios/storage/test_fio.py b/tests/unit/benchmark/scenarios/storage/test_fio.py
index 55e443885..17594b9f4 100644
--- a/tests/unit/benchmark/scenarios/storage/test_fio.py
+++ b/tests/unit/benchmark/scenarios/storage/test_fio.py
@@ -55,6 +55,20 @@ class FioTestCase(unittest.TestCase):
         self.assertIsNotNone(p.client)
         self.assertEqual(p.setup_done, True)
 
+    def test_fio_job_file_successful_setup(self, mock_ssh):
+
+        options = {
+            'job_file': 'job_file.ini',
+            'directory': '/FIO_Test'
+        }
+        args = {'options': options}
+        p = fio.Fio(args, self.ctx)
+        p.setup()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+        self.assertIsNotNone(p.client)
+        self.assertEqual(p.setup_done, True)
+
     def test_fio_successful_no_sla(self, mock_ssh):
 
         options = {
diff --git a/yardstick/benchmark/scenarios/storage/fio.py b/yardstick/benchmark/scenarios/storage/fio.py
index b99e34270..98fe26973 100644
--- a/yardstick/benchmark/scenarios/storage/fio.py
+++ b/yardstick/benchmark/scenarios/storage/fio.py
@@ -28,6 +28,14 @@ class Fio(base.Scenario):
         type:    string
         unit:    na
         default: /home/ubuntu/data.raw
+    job_file - fio job configuration file
+        type:    string
+        unit:    na
+        default: None
+    directory - mount directoey for test volume
+        type:    string
+        unit:    na
+        default: None
     bs - block size used for the io units
         type:    int
         unit:    bytes
@@ -71,20 +79,42 @@ class Fio(base.Scenario):
     def __init__(self, scenario_cfg, context_cfg):
         self.scenario_cfg = scenario_cfg
         self.context_cfg = context_cfg
+        self.options = self.scenario_cfg["options"]
         self.setup_done = False
 
     def setup(self):
         """scenario setup"""
-        self.target_script = pkg_resources.resource_filename(
-            "yardstick.benchmark.scenarios.storage",
-            Fio.TARGET_SCRIPT)
         host = self.context_cfg["host"]
 
         self.client = ssh.SSH.from_node(host, defaults={"user": "root"})
         self.client.wait(timeout=600)
 
-        # copy script to host
-        self.client._put_file_shell(self.target_script, '~/fio.sh')
+        self.job_file = self.options.get("job_file", None)
+
+        if self.job_file:
+            self.job_file_script = pkg_resources.resource_filename(
+                "yardstick.resources", 'files/' + self.job_file)
+
+            # copy script to host
+            self.client._put_file_shell(self.job_file_script, '~/job_file.ini')
+
+        else:
+            self.target_script = pkg_resources.resource_filename(
+                "yardstick.benchmark.scenarios.storage", Fio.TARGET_SCRIPT)
+
+            # copy script to host
+            self.client._put_file_shell(self.target_script, '~/fio.sh')
+
+        mount_dir = self.options.get("directory", None)
+
+        if mount_dir:
+            LOG.debug("Formating volume...")
+            self.client.execute("sudo mkfs.ext4 /dev/vdb")
+            cmd = "sudo mkdir %s" % mount_dir
+            self.client.execute(cmd)
+            LOG.debug("Mounting volume at: %s", mount_dir)
+            cmd = "sudo mount /dev/vdb %s" % mount_dir
+            self.client.execute(cmd)
 
         self.setup_done = True
 
@@ -92,57 +122,69 @@ class Fio(base.Scenario):
         """execute the benchmark"""
         default_args = "-ioengine=libaio -group_reporting -time_based -time_based " \
             "--output-format=json"
+        timeout = 3600
 
         if not self.setup_done:
             self.setup()
 
-        options = self.scenario_cfg["options"]
-        filename = options.get("filename", "/home/ubuntu/data.raw")
-        bs = options.get("bs", "4k")
-        iodepth = options.get("iodepth", "1")
-        rw = options.get("rw", "write")
-        ramp_time = options.get("ramp_time", 20)
-        size = options.get("size", "1g")
-        direct = options.get("direct", "1")
-        numjobs = options.get("numjobs", "1")
-        rwmixwrite = options.get("rwmixwrite", 50)
-        name = "yardstick-fio"
-        # if run by a duration runner
-        duration_time = self.scenario_cfg["runner"].get("duration", None) \
-            if "runner" in self.scenario_cfg else None
-        # if run by an arithmetic runner
-        arithmetic_time = options.get("duration", None)
-        if duration_time:
-            runtime = duration_time
-        elif arithmetic_time:
-            runtime = arithmetic_time
+        if self.job_file:
+            cmd = "sudo fio job_file.ini --output-format=json"
         else:
-            runtime = 30
+            filename = self.options.get("filename", "/home/ubuntu/data.raw")
+            bs = self.options.get("bs", "4k")
+            iodepth = self.options.get("iodepth", "1")
+            rw = self.options.get("rw", "write")
+            ramp_time = self.options.get("ramp_time", 20)
+            size = self.options.get("size", "1g")
+            direct = self.options.get("direct", "1")
+            numjobs = self.options.get("numjobs", "1")
+            rwmixwrite = self.options.get("rwmixwrite", 50)
+            name = "yardstick-fio"
+            # if run by a duration runner
+            duration_time = self.scenario_cfg["runner"].get("duration", None) \
+                if "runner" in self.scenario_cfg else None
+            # if run by an arithmetic runner
+            arithmetic_time = self.options.get("duration", None)
+            if duration_time:
+                runtime = duration_time
+            elif arithmetic_time:
+                runtime = arithmetic_time
+            else:
+                runtime = 30
+            # Set timeout, so that the cmd execution does not exit incorrectly
+            # when the test run time is last long
+            timeout = int(ramp_time) + int(runtime) + 600
+
+            cmd_args = "-filename=%s -direct=%s -bs=%s -iodepth=%s -rw=%s -rwmixwrite=%s " \
+                       "-size=%s -ramp_time=%s -numjobs=%s -runtime=%s -name=%s %s" \
+                       % (filename, direct, bs, iodepth, rw, rwmixwrite, size, ramp_time, numjobs,
+                          runtime, name, default_args)
+            cmd = "sudo bash fio.sh %s %s" % (filename, cmd_args)
 
-        cmd_args = "-filename=%s -direct=%s -bs=%s -iodepth=%s -rw=%s -rwmixwrite=%s " \
-                   "-size=%s -ramp_time=%s -numjobs=%s -runtime=%s -name=%s %s" \
-                   % (filename, direct, bs, iodepth, rw, rwmixwrite, size, ramp_time, numjobs,
-                      runtime, name, default_args)
-        cmd = "sudo bash fio.sh %s %s" % (filename, cmd_args)
         LOG.debug("Executing command: %s", cmd)
-        # Set timeout, so that the cmd execution does not exit incorrectly
-        # when the test run time is last long
-        timeout = int(ramp_time) + int(runtime) + 600
         status, stdout, stderr = self.client.execute(cmd, timeout=timeout)
         if status:
             raise RuntimeError(stderr)
 
         raw_data = jsonutils.loads(stdout)
 
-        # The bandwidth unit is KB/s, and latency unit is us
-        if rw in ["read", "randread", "rw", "randrw"]:
+        if self.job_file:
             result["read_bw"] = raw_data["jobs"][0]["read"]["bw"]
             result["read_iops"] = raw_data["jobs"][0]["read"]["iops"]
             result["read_lat"] = raw_data["jobs"][0]["read"]["lat"]["mean"]
-        if rw in ["write", "randwrite", "rw", "randrw"]:
             result["write_bw"] = raw_data["jobs"][0]["write"]["bw"]
             result["write_iops"] = raw_data["jobs"][0]["write"]["iops"]
             result["write_lat"] = raw_data["jobs"][0]["write"]["lat"]["mean"]
+        else:
+            # The bandwidth unit is KB/s, and latency unit is us
+            if rw in ["read", "randread", "rw", "randrw"]:
+                result["read_bw"] = raw_data["jobs"][0]["read"]["bw"]
+                result["read_iops"] = raw_data["jobs"][0]["read"]["iops"]
+                result["read_lat"] = raw_data["jobs"][0]["read"]["lat"]["mean"]
+            if rw in ["write", "randwrite", "rw", "randrw"]:
+                result["write_bw"] = raw_data["jobs"][0]["write"]["bw"]
+                result["write_iops"] = raw_data["jobs"][0]["write"]["iops"]
+                result["write_lat"] = raw_data["jobs"][0]["write"]["lat"]["mean"]
 
         if "sla" in self.scenario_cfg:
             sla_error = ""