Merge "SPEC cpu2006 test case for VM"

3 files changed, 321 insertions, 0 deletions

diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc084.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc084.yaml
new file mode 100644
index 000000000..472aabe07
--- /dev/null
+++ b/tests/opnfv/test_cases/opnfv_yardstick_tc084.yaml
@@ -0,0 +1,67 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+
+schema: "yardstick:task:0.1"
+description: >
+    Yardstick TC084 config file;
+    Measure CPU performance using SPEC CPU2006 in Virtual machines;
+
+{% set provider = provider or none %}
+{% set physical_network = physical_network or 'physnet1' %}
+{% set segmentation_id = segmentation_id or none %}
+
+{% set benchmark = benchmark or 'int' %}
+{% set runspec_iterations = runspec_iterations or 1 %}
+{% set runspec_tune = runspec_tune or 'base' %}
+{% set runspec_size = runspec_size or 'ref' %}
+{% set runspec_rate = runspec_rate or 1 %}
+
+scenarios:
+-
+  type: SpecCPU2006_for_VM
+
+  options:
+      SPECint_benchmark: {{benchmark}}
+      runspec_iterations: {{runspec_iterations}}
+      runspec_tune: {{runspec_tune}}
+      runspec_size: {{runspec_size}}
+      runspec_rate: {{runspec_rate}}
+
+  host: spec.yardstick-TC084
+
+  runner:
+    type: Iteration
+    iterations: 1
+
+context:
+  name: yardstick-TC084
+  image: yardstick-image
+  flavor: yardstick-flavor
+  user: ubuntu
+
+  placement_groups:
+    pgrp1:
+      policy: "availability"
+
+  servers:
+    spec:
+      floating_ip: true
+      placement: "pgrp1"
+
+  networks:
+    test:
+      cidr: '10.0.1.0/24'
+      {% if provider == "vlan" %}
+      provider: {{provider}}
+      physical_network: {{physical_network}}
+        {% if segmentation_id %}
+      segmentation_id: {{segmentation_id}}
+        {% endif %}
+      {% endif %}
diff --git a/tests/unit/benchmark/scenarios/compute/test_spec_cpu_for_vm.py b/tests/unit/benchmark/scenarios/compute/test_spec_cpu_for_vm.py
new file mode 100644
index 000000000..c428e1fb8
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/compute/test_spec_cpu_for_vm.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.compute.spec_cpu_for_vm.SpecCPUforVM
+
+from __future__ import absolute_import
+
+import unittest
+
+import mock
+
+from yardstick.benchmark.scenarios.compute import spec_cpu_for_vm
+
+
+@mock.patch('yardstick.benchmark.scenarios.compute.spec_cpu_for_vm.ssh')
+class SpecCPUforVMTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.ctx = {
+            'host': {
+                'ip': '172.16.0.137',
+                'user': 'root',
+                'key_filename': "mykey.key"
+            }
+        }
+
+        self.result = {}
+
+    def test_spec_cpu_successful_setup(self, mock_ssh):
+
+        options = {
+            "SPECint_benchmark": "perlbench",
+            "runspec_tune": "all",
+            "output_format": "all",
+            "runspec_iterations": "1",
+            "runspec_size": "test"
+        }
+        args = {"options": options}
+        s = spec_cpu_for_vm.SpecCPUforVM(args, self.ctx)
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+        s.setup()
+        self.assertIsNotNone(s.client)
+        self.assertTrue(s.setup_done, True)
+
+    def test_spec_cpu_successful__run_no_sla(self, mock_ssh):
+
+        options = {
+            "SPECint_benchmark": "perlbench",
+            "runspec_tune": "all",
+            "output_format": "all"
+        }
+        args = {"options": options}
+        s = spec_cpu_for_vm.SpecCPUforVM(args, self.ctx)
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+        mock_ssh.SSH.from_node().get.return_value = (0, '', '')
+        s.run(self.result)
+        expected_result = {'SPEC_CPU_result': ''}
+        self.assertEqual(self.result, expected_result)
+
+    def test_spec_cpu_unsuccessful_script_error(self, mock_ssh):
+        options = {
+            "benchmark_subset": "int"
+        }
+        args = {"options": options}
+        s = spec_cpu_for_vm.SpecCPUforVM(args, self.ctx)
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', 'FOOBAR')
+        self.assertRaises(RuntimeError, s.run, self.result)
+
+def main():
+    unittest.main()
+
+if __name__ == '__main__':
+    main()
diff --git a/yardstick/benchmark/scenarios/compute/spec_cpu_for_vm.py b/yardstick/benchmark/scenarios/compute/spec_cpu_for_vm.py
new file mode 100644
index 000000000..36489b132
--- /dev/null
+++ b/yardstick/benchmark/scenarios/compute/spec_cpu_for_vm.py
@@ -0,0 +1,170 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from __future__ import absolute_import
+
+import logging
+import pkg_resources
+import os
+
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+from yardstick.common.constants import YARDSTICK_ROOT_PATH
+
+LOG = logging.getLogger(__name__)
+
+
+class SpecCPUforVM(base.Scenario):
+    """Spec CPU2006 benchmark for Virtual Machine
+
+    Parameters
+        benchmark_subset - Specifies a subset of SPEC CPU2006 benchmarks to run
+            type:       string
+            unit:       na
+            default:    na
+
+        SPECint_benchmark - A SPECint benchmark to run
+            type:       string
+            unit:       na
+            default:    na
+
+        SPECfp_benchmark - A SPECfp benchmark to run
+            type:       string
+            unit:       na
+            default:    na
+
+        output_format - Desired report format
+            type:       string
+            unit:       na
+            default:    na
+
+        runspec_config - SPEC CPU2006 config file provided to the runspec binary
+            type:       string
+            unit:       na
+            default:    "Example-linux64-amd64-gcc43+.cfg"
+
+        runspec_iterations - The number of benchmark iterations to execute.
+                             For a reportable run, must be 3.
+            type:       int
+            unit:       na
+            default:    na
+
+        runspec_tune - Tuning to use (base, peak, or all). For a reportable run, must be either
+                       base or all. Reportable runs do base first, then (optionally) peak.
+            type:       string
+            unit:       na
+            default:    na
+
+        runspec_size - Size of input data to run (test, train, or ref). Reportable runs ensure
+                       that your binaries can produce correct results with the test and train
+                       workloads.
+            type:       string
+            unit:       na
+            default:    na
+    """
+    __scenario_type__ = "SpecCPU2006_for_VM"
+    CPU2006_ISO = "cpu2006-1.2.iso"
+    CPU2006_DIR = "~/cpu2006"
+    CPU2006_RESULT_FILE = os.path.join(CPU2006_DIR, "result/CINT2006.001.ref.txt")
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.setup_done = False
+        self.options = self.scenario_cfg['options']
+
+    def setup(self):
+        """scenario setup"""
+        host = self.context_cfg['host']
+        LOG.info("user:%s, host:%s", host['user'], host['ip'])
+        self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
+        self.client.wait(timeout=600)
+
+        spec_cpu_iso = os.path.join(YARDSTICK_ROOT_PATH,
+                                    "yardstick/resources/files/",
+                                    self.CPU2006_ISO)
+
+        self.client.put(spec_cpu_iso, "~/cpu2006-1.2.iso")
+        self.client.execute("sudo mount -t iso9660 -o ro,exec ~/cpu2006-1.2.iso /mnt")
+        self.client.execute("/mnt/install.sh -fd ~/cpu2006")
+
+        if "runspec_config" in self.options:
+            self.runspec_config = self.options["runspec_config"]
+
+            self.runspec_config_file = pkg_resources.resource_filename(
+                "yardstick.resources", 'files/' + self.runspec_config)
+
+            # copy SPEC CPU2006 config file to host if given
+            cfg_path = os.path.join(self.CPU2006_DIR,
+                                    'config/yardstick_spec_cpu2006.cfg')
+            self.client._put_file_shell(self.runspec_config_file, cfg_path)
+        else:
+            self.runspec_config = "Example-linux64-amd64-gcc43+.cfg"
+
+        self.setup_done = True
+
+    def run(self, result):
+        """execute the benchmark"""
+
+        if not self.setup_done:
+            self.setup()
+
+        cmd = "cd %s && . ./shrc && runspec --config %s" % (
+            self.CPU2006_DIR, self.runspec_config)
+        cmd_args = ""
+
+        if "rate" in self.options:
+            cmd_args += " --rate %s" % self.options["runspec_rate"]
+
+        if "output_format" in self.options:
+            cmd_args += " --output_format %s" % self.options["output_format"]
+
+        if "runspec_tune" in self.options:
+            cmd_args += " --tune %s" % self.options["runspec_tune"]
+
+        benchmark_subset = self.options.get('benchmark_subset', None)
+        specint_benchmark = self.options.get('SPECint_benchmark', None)
+        specfp_benchmark = self.options.get('SPECfp_benchmark', None)
+
+        if benchmark_subset:
+            cmd_args += " %s" % benchmark_subset
+        else:
+            cmd_args += " --noreportable"
+
+            if "runspec_iterations" in self.options:
+                cmd_args += " --iterations %s" % self.options["runspec_iterations"]
+
+            if "runspec_size" in self.options:
+                cmd_args += " --size %s" % self.options["runspec_size"]
+
+            if specint_benchmark:
+                cmd_args += " %s" % specint_benchmark
+
+            if specfp_benchmark:
+                cmd_args += " %s" % specfp_benchmark
+
+        cmd += "%s" % cmd_args
+
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.client.execute(cmd, timeout=86400)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "cat %s" % self.CPU2006_RESULT_FILE
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.client.execute(cmd, timeout=30)
+        if status:
+            raise RuntimeError(stderr)
+        if stdout:
+            LOG.info("SPEC CPU2006 result is:\n%s", stdout)
+
+        result.update({"SPEC_CPU_result": stdout})
+        # fetch SPEC CPU2006 result files
+        self.client.get('~/cpu2006/result', '/tmp/')
+        LOG.info('SPEC CPU2006 benchmark completed, please find benchmark reports \
+                  at /tmp/result directory')