Merge "Add spec cpu2006 test case"

11 files changed, 538 insertions, 0 deletions

diff --git a/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml
new file mode 100644
index 000000000..7f72e1394
--- /dev/null
+++ b/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml
@@ -0,0 +1,15 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: archive_spec_cpu2006_result
+  archive:
+    path: /usr/cpu2006/result
+    dest: /usr/cpu2006/spec_cpu2006_result.zip
+  become: true
diff --git a/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml
new file mode 100644
index 000000000..370ec7a0c
--- /dev/null
+++ b/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml
@@ -0,0 +1,16 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: fetch_spec_cpu2006_result
+  fetch:
+    src: /usr/cpu2006/spec_cpu2006_result.zip
+    dest: /tmp/
+    flat: yes
+  become: true
diff --git a/ansible/roles/install_spec_cpu2006/tasks/main.yaml b/ansible/roles/install_spec_cpu2006/tasks/main.yaml
new file mode 100644
index 000000000..baa9f6ca0
--- /dev/null
+++ b/ansible/roles/install_spec_cpu2006/tasks/main.yaml
@@ -0,0 +1,24 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: upload SPEC CPU2006 ISO
+  copy:
+    src: /home/opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso
+    dest: ~/cpu2006-1.2.iso
+  become: true
+
+- name: install SPEC CPU2006
+  shell:
+    mount -t iso9660 -o ro,exec ~/cpu2006-1.2.iso /mnt;
+    cd /mnt;
+    ./install.sh -fd /usr/cpu2006;
+  args:
+      executable: /bin/bash
+  become: true
diff --git a/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml
new file mode 100644
index 000000000..86dcc1aeb
--- /dev/null
+++ b/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml
@@ -0,0 +1,16 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: unarchive_spec_cpu2006_result
+  unarchive:
+    src: /tmp/spec_cpu2006_result.zip
+    dest: /tmp/
+    remote_src: True
+  become: true
diff --git a/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml b/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml
new file mode 100644
index 000000000..72d8c93ee
--- /dev/null
+++ b/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml
@@ -0,0 +1,28 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: remove SPEC CPU 2006
+  file:
+    path: /usr/cpu2006
+    state: absent
+  become: true
+
+- name: umount SPEC CPU 2006 ISO
+  shell:
+    umount -l /mnt;
+  args:
+      executable: /bin/bash
+  become: true
+
+- name: remove SPEC CPU 2006 ISO
+  file:
+    path: ~/cpu2006-1.2.iso
+    state: absent
+  become: true
diff --git a/ansible/spec_cpu2006_install.yaml b/ansible/spec_cpu2006_install.yaml
new file mode 100644
index 000000000..470f5c8d8
--- /dev/null
+++ b/ansible/spec_cpu2006_install.yaml
@@ -0,0 +1,13 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- hosts: spec_target
+  roles:
+    - install_spec_cpu2006
diff --git a/ansible/spec_cpu2006_uninstall.yaml b/ansible/spec_cpu2006_uninstall.yaml
new file mode 100644
index 000000000..4991c98b0
--- /dev/null
+++ b/ansible/spec_cpu2006_uninstall.yaml
@@ -0,0 +1,22 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- hosts: spec_target
+  roles:
+    - archive_spec_cpu2006_result
+    - fetch_spec_cpu2006_result
+
+- hosts: localhost
+  roles:
+    - unarchive_spec_cpu2006_result
+
+- hosts: spec_target
+  roles:
+    - uninstall_spec_cpu2006
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc078.rst b/docs/testing/user/userguide/opnfv_yardstick_tc078.rst
new file mode 100644
index 000000000..560d55e07
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc078.rst
@@ -0,0 +1,133 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC078
+*************************************
+
+.. _spec_cpu2006: https://www.spec.org/cpu2006/
+
++-----------------------------------------------------------------------------+
+|Compute Performance                                                          |
+|                                                                             |
++--------------+--------------------------------------------------------------+
+|test case id  | OPNFV_YARDSTICK_TC078_SPEC CPU 2006                          |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|metric        | compute-intensive performance                                |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test purpose  | The purpose of TC078 is to evaluate the IaaS compute         |
+|              | performance by using SPEC CPU 2006 benchmark. The SPEC CPU   |
+|              | 2006 benchmark has several different ways to measure         |
+|              | computer performance. One way is to measure how fast the     |
+|              | computer completes a single task; this is called a speed     |
+|              | measurement. Another way is to measure how many tasks        |
+|              | computer can accomplish in a certain amount of time; this is |
+|              | called a throughput, capacity or rate measurement.           |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test tool     | SPEC CPU 2006                                                |
+|              |                                                              |
+|              | The SPEC CPU 2006 benchmark is SPEC's industry-standardized, |
+|              | CPU-intensive benchmark suite, stressing a system's          |
+|              | processor, memory subsystem and compiler. This benchmark     |
+|              | suite includes the SPECint benchmarks and the SPECfp         |
+|              | benchmarks. The SPECint 2006 benchmark contains 12 different |
+|              | enchmark tests and the SPECfp 2006 benchmark contains 19     |
+|              | different benchmark tests.                                   |
+|              |                                                              |
+|              | SPEC CPU 2006 is not always part of a Linux distribution.    |
+|              | SPEC requires that users purchase a license and agree with   |
+|              | their terms and conditions. For this test case, users must   |
+|              | manually download cpu2006-1.2.iso from the SPEC website and  |
+|              | save it under the yardstick/resources folder (e.g. /home/    |
+|              | opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso)   |
+|              | SPEC CPU® 2006 benchmark is available for purchase via the   |
+|              | SPEC order form (https://www.spec.org/order.html).           |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test          | This test case uses SPEC CPU 2006 benchmark to measure       |
+|description   | compute-intensive performance of hosts.                      |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|configuration | file: spec_cpu.yaml (in the 'samples' directory)             |
+|              |                                                              |
+|              | benchmark_subset is set to int.                              |
+|              |                                                              |
+|              | SLA is not available in this test case.                      |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different:                       |
+|              |                                                              |
+|              |  * benchmark_subset - a subset of SPEC CPU2006 benchmarks to |
+|              |    run;                                                      |
+|              |  * SPECint_benchmark - a SPECint benchmark to run;           |
+|              |  * SPECint_benchmark - a SPECfp benchmark to run;            |
+|              |  * output_format - desired report format;                    |
+|              |  * runspec_config - SPEC CPU2006 config file provided to the |
+|              |    runspec binary;                                           |
+|              |  * runspec_iterations - the number of benchmark iterations   |
+|              |    to execute. For a reportable run, must be 3;              |
+|              |  * runspec_tune - tuning to use (base, peak, or all). For a  |
+|              |    reportable run, must be either base or all. Reportable    |
+|              |    runs do base first, then (optionally) peak;               |
+|              |  * runspec_size - size of input data to run (test, train, or |
+|              |    ref). Reportable runs ensure that your binaries can       |
+|              |    produce correct results with the test and train workloads |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|usability     | This test case is used for executing SPEC CPU 2006 benchmark |
+|              | physical servers. The SPECint 2006 benchmark takes           |
+|              | approximately 5 hours.                                       |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|references    | spec_cpu2006_                                                |
+|              |                                                              |
+|              | ETSI-NFV-TST001                                              |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|pre-test      | To run and install SPEC CPU2006, the following are required: |
+|conditions    |  * For SPECint2006: Both C99 and C++98 compilers;            |
+|              |  * For SPECfp2006: All three of C99, C++98 and Fortran-95    |
+|              |    compilers;                                                |
+|              |  * At least 8GB of disk space availabile on the system.      |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result                              |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 1        | cpu2006-1.2.iso has been saved under the yardstick/resources |
+|              | folder (e.g. /home/opnfv/repos/yardstick/yardstick/resources |
+|              | /cpu2006-1.2.iso). Additional, to use your custom runspec    |
+|              | config file you can save it under the yardstick/resources/   |
+|              | files folder and specify the config file name in the         |
+|              | runspec_config parameter.                                    |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 2        | Upload SPEC CPU2006 ISO to the target server and install     |
+|              | SPEC CPU2006 via ansible.                                    |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 3        | Yardstick is connected with the target server by using ssh.  |
+|              | If custom runspec config file is used, this file is copyied  |
+|              | from yardstick to the target server via the ssh tunnel.      |
+--------------+---------------------------------------------------------------+
+|step 4        | SPEC CPU2006 benchmark is invoked and SPEC CPU 2006 metrics  |
+|              | are generated.                                               |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 5        | Text, HTML, CSV, PDF, and Configuration file outputs for the |
+|              | SPEC CPU 2006 metrics are fetch from the server and stored   |
+|              | under /tmp/result folder.                                    |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|step 6        | uninstall SPEC CPU2006 and remove cpu2006-1.2.iso from the   |
+|              | target server .                                              |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
+|test verdict  | None. SPEC CPU2006 results are collected and stored.         |
+|              |                                                              |
++--------------+--------------------------------------------------------------+
diff --git a/samples/spec_cpu.yaml b/samples/spec_cpu.yaml
new file mode 100644
index 000000000..227c3e9d2
--- /dev/null
+++ b/samples/spec_cpu.yaml
@@ -0,0 +1,44 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+schema: "yardstick:task:0.1"
+
+{% set file = file or "/etc/yardstick/pod.yaml" %}
+
+scenarios:
+-
+  type: SpecCPU2006
+
+  options:
+      benchmark_subset: int
+      #runspec_tune: all
+      #output_format: all
+      #SPECint_benchmark: perlbench
+      #SPECfp_benchmark: bwaves
+      #runspec_config: yardstick_spec_cpu2006.cfg
+      #runspec_iterations: 1
+      #runspec_tune: base
+      #runspec_size: test
+
+  host: node1.spec_cpu2006
+
+  runner:
+    type: Iteration
+    iterations: 1
+
+context:
+  type: Node
+  name: spec_cpu2006
+  file: {{ file }}
+
+  env:
+    type: ansible
+    setup: spec_cpu2006_install.yaml
+    teardown: spec_cpu2006_uninstall.yaml
diff --git a/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py b/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py
new file mode 100644
index 000000000..40423b9da
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.compute.spec_cpu.SpecCPU
+
+from __future__ import absolute_import
+
+import unittest
+
+import mock
+
+from yardstick.common import utils
+from yardstick.benchmark.scenarios.compute import spec_cpu
+
+
+@mock.patch('yardstick.benchmark.scenarios.compute.spec_cpu.ssh')
+class SpecCPUTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.ctx = {
+            'host': {
+                'ip': '172.16.0.137',
+                'user': 'root',
+                'key_filename': "mykey.key"
+            }
+        }
+
+        self.result = {}
+
+    def test_spec_cpu_successful_setup(self, mock_ssh):
+
+        options = {
+            "SPECint_benchmark": "perlbench",
+            "runspec_tune": "all",
+            "output_format": "all",
+            "runspec_iterations": "1",
+            "runspec_tune": "base",
+            "runspec_size": "test"
+        }
+        args = {"options": options}
+        s = spec_cpu.SpecCPU(args, self.ctx)
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+        s.setup()
+        self.assertIsNotNone(s.client)
+        self.assertTrue(s.setup_done, True)
+
+    def test_spec_cpu_successful__run_no_sla(self, mock_ssh):
+
+        options = {
+            "SPECint_benchmark": "perlbench",
+            "runspec_tune": "all",
+            "output_format": "all"
+        }
+        args = {"options": options}
+        s = spec_cpu.SpecCPU(args, self.ctx)
+
+        sample_output = ''
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+        s.run(self.result)
+        expected_result = {}
+        self.assertEqual(self.result, expected_result)
+
+    def test_ramspeed_unsuccessful_script_error(self, mock_ssh):
+        options = {
+            "benchmark_subset": "int"
+        }
+        args = {"options": options}
+        s = spec_cpu.SpecCPU(args, self.ctx)
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', 'FOOBAR')
+        self.assertRaises(RuntimeError, s.run, self.result)
+
+def main():
+    unittest.main()
+
+if __name__ == '__main__':
+    main()
diff --git a/yardstick/benchmark/scenarios/compute/spec_cpu.py b/yardstick/benchmark/scenarios/compute/spec_cpu.py
new file mode 100644
index 000000000..520618631
--- /dev/null
+++ b/yardstick/benchmark/scenarios/compute/spec_cpu.py
@@ -0,0 +1,141 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from __future__ import absolute_import
+
+import logging
+import pkg_resources
+
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class SpecCPU(base.Scenario):
+    """Spec CPU2006 benchmark
+
+    Parameters
+        benchmark_subset - Specifies a subset of SPEC CPU2006 benchmarks to run
+            type:       string
+            unit:       na
+            default:    na
+
+        SPECint_benchmark - A SPECint benchmark to run
+            type:       string
+            unit:       na
+            default:    na
+
+        SPECint_benchmark - A SPECfp benchmark to run
+            type:       string
+            unit:       na
+            default:    na
+
+        output_format - Desired report format
+            type:       string
+            unit:       na
+            default:    na
+
+        runspec_config - SPEC CPU2006 config file provided to the runspec binary
+            type:       string
+            unit:       na
+            default:    "Example-linux64-amd64-gcc43+.cfg"
+
+        runspec_iterations - The number of benchmark iterations to execute.
+                             For a reportable run, must be 3.
+            type:       int
+            unit:       na
+            default:    na
+
+        runspec_tune - Tuning to use (base, peak, or all). For a reportable run, must be either
+                       base or all. Reportable runs do base first, then (optionally) peak.
+            type:       string
+            unit:       na
+            default:    na
+
+        runspec_size - Size of input data to run (test, train, or ref). Reportable runs ensure
+                       that your binaries can produce correct results with the test and train
+                       workloads.
+            type:       string
+            unit:       na
+            default:    na
+    """
+    __scenario_type__ = "SpecCPU2006"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.setup_done = False
+        self.options = self.scenario_cfg['options']
+
+    def setup(self):
+        """scenario setup"""
+        host = self.context_cfg['host']
+        LOG.info("user:%s, host:%s", host['user'], host['ip'])
+        self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
+        self.client.wait(timeout=600)
+
+        if "runspec_config" in self.options:
+            self.runspec_config = self.options["runspec_config"]
+
+            self.runspec_config_file = pkg_resources.resource_filename(
+                "yardstick.resources", 'files/' + self.runspec_config)
+
+            # copy SPEC CPU2006 config file to host if given
+            self.client._put_file_shell(self.runspec_config_file,
+                                        '/usr/cpu2006/config/yardstick_spec_cpu2006.cfg')
+        else:
+            self.runspec_config = "Example-linux64-amd64-gcc43+.cfg"
+
+        self.setup_done = True
+
+    def run(self, result):
+        """execute the benchmark"""
+
+        if not self.setup_done:
+            self.setup()
+
+        cmd = "cd /usr/cpu2006/ && . ./shrc && runspec --config %s" % self.runspec_config
+        cmd_args = ""
+
+        if "output_format" in self.options:
+            cmd_args += " --output_format %s" % self.options["output_format"]
+
+        if "runspec_tune" in self.options:
+            cmd_args += " --tune %s" % self.options["runspec_tune"]
+
+        benchmark_subset = self.options.get('benchmark_subset', None)
+        specint_benchmark = self.options.get('SPECint_benchmark', None)
+        specfp_benchmark = self.options.get('SPECfp_benchmark', None)
+
+        if benchmark_subset:
+            cmd_args += " %s" % benchmark_subset
+        else:
+            cmd_args += " --noreportable"
+
+            if "runspec_iterations" in self.options:
+                cmd_args += " --iterations %s" % self.options["runspec_iterations"]
+
+            if "runspec_size" in self.options:
+                cmd_args += " --size %s" % self.options["runspec_size"]
+
+            if specint_benchmark:
+                cmd_args += " %s" % specint_benchmark
+
+            if specfp_benchmark:
+                cmd_args += " %s" % specfp_benchmark
+
+        cmd += "%s" % cmd_args
+
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.client.execute(cmd, timeout=86400)
+        if status:
+            raise RuntimeError(stderr)
+
+        LOG.info('SPEC CPU2006 benchmark completed, please find benchmark reports \
+                  at /tmp/result directory')