summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml15
-rw-r--r--ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml16
-rw-r--r--ansible/roles/install_spec_cpu2006/tasks/main.yaml24
-rw-r--r--ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml16
-rw-r--r--ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml28
-rw-r--r--ansible/spec_cpu2006_install.yaml13
-rw-r--r--ansible/spec_cpu2006_uninstall.yaml22
-rw-r--r--api/resources/v1/env.py2
-rw-r--r--api/resources/v2/testcases.py7
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc078.rst133
-rwxr-xr-xinstall.sh20
-rwxr-xr-xnsb_setup.sh2
-rw-r--r--requirements.txt1
-rw-r--r--samples/spec_cpu.yaml44
-rw-r--r--tests/unit/apiserver/resources/test_env_action.py4
-rw-r--r--tests/unit/benchmark/scenarios/compute/test_spec_cpu.py86
-rw-r--r--yardstick/benchmark/contexts/model.py7
-rw-r--r--yardstick/benchmark/core/plugin.py2
-rw-r--r--yardstick/benchmark/core/task.py4
-rw-r--r--yardstick/benchmark/core/testcase.py2
-rw-r--r--yardstick/benchmark/scenarios/availability/attacker/baseattacker.py2
-rw-r--r--yardstick/benchmark/scenarios/availability/monitor/basemonitor.py2
-rw-r--r--yardstick/benchmark/scenarios/availability/operation/baseoperation.py2
-rw-r--r--yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py2
-rw-r--r--yardstick/benchmark/scenarios/compute/spec_cpu.py141
-rw-r--r--yardstick/common/template_format.py1
26 files changed, 576 insertions, 22 deletions
diff --git a/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml
new file mode 100644
index 000000000..7f72e1394
--- /dev/null
+++ b/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml
@@ -0,0 +1,15 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: archive_spec_cpu2006_result
+ archive:
+ path: /usr/cpu2006/result
+ dest: /usr/cpu2006/spec_cpu2006_result.zip
+ become: true
diff --git a/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml
new file mode 100644
index 000000000..370ec7a0c
--- /dev/null
+++ b/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml
@@ -0,0 +1,16 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: fetch_spec_cpu2006_result
+ fetch:
+ src: /usr/cpu2006/spec_cpu2006_result.zip
+ dest: /tmp/
+ flat: yes
+ become: true
diff --git a/ansible/roles/install_spec_cpu2006/tasks/main.yaml b/ansible/roles/install_spec_cpu2006/tasks/main.yaml
new file mode 100644
index 000000000..baa9f6ca0
--- /dev/null
+++ b/ansible/roles/install_spec_cpu2006/tasks/main.yaml
@@ -0,0 +1,24 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: upload SPEC CPU2006 ISO
+ copy:
+ src: /home/opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso
+ dest: ~/cpu2006-1.2.iso
+ become: true
+
+- name: install SPEC CPU2006
+ shell:
+ mount -t iso9660 -o ro,exec ~/cpu2006-1.2.iso /mnt;
+ cd /mnt;
+ ./install.sh -fd /usr/cpu2006;
+ args:
+ executable: /bin/bash
+ become: true
diff --git a/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml
new file mode 100644
index 000000000..86dcc1aeb
--- /dev/null
+++ b/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml
@@ -0,0 +1,16 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: unarchive_spec_cpu2006_result
+ unarchive:
+ src: /tmp/spec_cpu2006_result.zip
+ dest: /tmp/
+ remote_src: True
+ become: true
diff --git a/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml b/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml
new file mode 100644
index 000000000..72d8c93ee
--- /dev/null
+++ b/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml
@@ -0,0 +1,28 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: remove SPEC CPU 2006
+ file:
+ path: /usr/cpu2006
+ state: absent
+ become: true
+
+- name: umount SPEC CPU 2006 ISO
+ shell:
+ umount -l /mnt;
+ args:
+ executable: /bin/bash
+ become: true
+
+- name: remove SPEC CPU 2006 ISO
+ file:
+ path: ~/cpu2006-1.2.iso
+ state: absent
+ become: true
diff --git a/ansible/spec_cpu2006_install.yaml b/ansible/spec_cpu2006_install.yaml
new file mode 100644
index 000000000..470f5c8d8
--- /dev/null
+++ b/ansible/spec_cpu2006_install.yaml
@@ -0,0 +1,13 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- hosts: spec_target
+ roles:
+ - install_spec_cpu2006
diff --git a/ansible/spec_cpu2006_uninstall.yaml b/ansible/spec_cpu2006_uninstall.yaml
new file mode 100644
index 000000000..4991c98b0
--- /dev/null
+++ b/ansible/spec_cpu2006_uninstall.yaml
@@ -0,0 +1,22 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- hosts: spec_target
+ roles:
+ - archive_spec_cpu2006_result
+ - fetch_spec_cpu2006_result
+
+- hosts: localhost
+ roles:
+ - unarchive_spec_cpu2006_result
+
+- hosts: spec_target
+ roles:
+ - uninstall_spec_cpu2006
diff --git a/api/resources/v1/env.py b/api/resources/v1/env.py
index 8943db3d1..8367fa9eb 100644
--- a/api/resources/v1/env.py
+++ b/api/resources/v1/env.py
@@ -393,7 +393,7 @@ class V1Env(ApiResource):
return result_handler(consts.API_ERROR, 'file must be provided')
LOG.info('Checking file')
- data = yaml.load(pod_file.read())
+ data = yaml.safe_load(pod_file.read())
if not isinstance(data, collections.Mapping):
return result_handler(consts.API_ERROR, 'invalid yaml file')
diff --git a/api/resources/v2/testcases.py b/api/resources/v2/testcases.py
index b47a8f6b7..316ef2664 100644
--- a/api/resources/v2/testcases.py
+++ b/api/resources/v2/testcases.py
@@ -10,6 +10,8 @@ import logging
import errno
import os
+import jinja2schema
+
from api import ApiResource
from yardstick.common.utils import result_handler
from yardstick.common import constants as consts
@@ -56,7 +58,10 @@ class V2Testcase(ApiResource):
if e.errno == errno.ENOENT:
return result_handler(consts.API_ERROR, 'case does not exist')
- return result_handler(consts.API_SUCCESS, {'testcase': data})
+ options = {k: {'description': '', 'type': v.__class__.__name__}
+ for k, v in jinja2schema.infer(data).items()}
+
+ return result_handler(consts.API_SUCCESS, {'testcase': data, 'args': options})
def delete(self, case_name):
case_path = os.path.join(consts.TESTCASE_DIR, '{}.yaml'.format(case_name))
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc078.rst b/docs/testing/user/userguide/opnfv_yardstick_tc078.rst
new file mode 100644
index 000000000..560d55e07
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc078.rst
@@ -0,0 +1,133 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC078
+*************************************
+
+.. _spec_cpu2006: https://www.spec.org/cpu2006/
+
++-----------------------------------------------------------------------------+
+|Compute Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC078_SPEC CPU 2006 |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | compute-intensive performance |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC078 is to evaluate the IaaS compute |
+| | performance by using SPEC CPU 2006 benchmark. The SPEC CPU |
+| | 2006 benchmark has several different ways to measure |
+| | computer performance. One way is to measure how fast the |
+| | computer completes a single task; this is called a speed |
+| | measurement. Another way is to measure how many tasks |
+| | computer can accomplish in a certain amount of time; this is |
+| | called a throughput, capacity or rate measurement. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | SPEC CPU 2006 |
+| | |
+| | The SPEC CPU 2006 benchmark is SPEC's industry-standardized, |
+| | CPU-intensive benchmark suite, stressing a system's |
+| | processor, memory subsystem and compiler. This benchmark |
+| | suite includes the SPECint benchmarks and the SPECfp |
+| | benchmarks. The SPECint 2006 benchmark contains 12 different |
+| | enchmark tests and the SPECfp 2006 benchmark contains 19 |
+| | different benchmark tests. |
+| | |
+| | SPEC CPU 2006 is not always part of a Linux distribution. |
+| | SPEC requires that users purchase a license and agree with |
+| | their terms and conditions. For this test case, users must |
+| | manually download cpu2006-1.2.iso from the SPEC website and |
+| | save it under the yardstick/resources folder (e.g. /home/ |
+| | opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso) |
+| | SPEC CPU® 2006 benchmark is available for purchase via the |
+| | SPEC order form (https://www.spec.org/order.html). |
+| | |
++--------------+--------------------------------------------------------------+
+|test | This test case uses SPEC CPU 2006 benchmark to measure |
+|description | compute-intensive performance of hosts. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: spec_cpu.yaml (in the 'samples' directory) |
+| | |
+| | benchmark_subset is set to int. |
+| | |
+| | SLA is not available in this test case. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * benchmark_subset - a subset of SPEC CPU2006 benchmarks to |
+| | run; |
+| | * SPECint_benchmark - a SPECint benchmark to run; |
+| | * SPECint_benchmark - a SPECfp benchmark to run; |
+| | * output_format - desired report format; |
+| | * runspec_config - SPEC CPU2006 config file provided to the |
+| | runspec binary; |
+| | * runspec_iterations - the number of benchmark iterations |
+| | to execute. For a reportable run, must be 3; |
+| | * runspec_tune - tuning to use (base, peak, or all). For a |
+| | reportable run, must be either base or all. Reportable |
+| | runs do base first, then (optionally) peak; |
+| | * runspec_size - size of input data to run (test, train, or |
+| | ref). Reportable runs ensure that your binaries can |
+| | produce correct results with the test and train workloads |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is used for executing SPEC CPU 2006 benchmark |
+| | physical servers. The SPECint 2006 benchmark takes |
+| | approximately 5 hours. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | spec_cpu2006_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | To run and install SPEC CPU2006, the following are required: |
+|conditions | * For SPECint2006: Both C99 and C++98 compilers; |
+| | * For SPECfp2006: All three of C99, C++98 and Fortran-95 |
+| | compilers; |
+| | * At least 8GB of disk space availabile on the system. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | cpu2006-1.2.iso has been saved under the yardstick/resources |
+| | folder (e.g. /home/opnfv/repos/yardstick/yardstick/resources |
+| | /cpu2006-1.2.iso). Additional, to use your custom runspec |
+| | config file you can save it under the yardstick/resources/ |
+| | files folder and specify the config file name in the |
+| | runspec_config parameter. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Upload SPEC CPU2006 ISO to the target server and install |
+| | SPEC CPU2006 via ansible. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | Yardstick is connected with the target server by using ssh. |
+| | If custom runspec config file is used, this file is copyied |
+| | from yardstick to the target server via the ssh tunnel. |
+--------------+---------------------------------------------------------------+
+|step 4 | SPEC CPU2006 benchmark is invoked and SPEC CPU 2006 metrics |
+| | are generated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 5 | Text, HTML, CSV, PDF, and Configuration file outputs for the |
+| | SPEC CPU 2006 metrics are fetch from the server and stored |
+| | under /tmp/result folder. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 6 | uninstall SPEC CPU2006 and remove cpu2006-1.2.iso from the |
+| | target server . |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. SPEC CPU2006 results are collected and stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/install.sh b/install.sh
index 8a5050a61..4ca72a757 100755
--- a/install.sh
+++ b/install.sh
@@ -14,6 +14,7 @@ DOCKER_ARCH="$(uname -m)"
UBUNTU_PORTS_URL="http://ports.ubuntu.com/ubuntu-ports/"
UBUNTU_ARCHIVE_URL="http://archive.ubuntu.com/ubuntu/"
+source /etc/os-release
source_file=/etc/apt/sources.list
if [[ "${DOCKER_ARCH}" == "aarch64" ]]; then
@@ -33,18 +34,25 @@ else
fi
sed -i -e 's/^deb-src /# deb-src /g' "${source_file}"
-echo "APT::Default-Release \"trusty\";" > /etc/apt/apt.conf.d/default-distro
+
+VERSION_CODENAME=${VERSION_CODENAME:-trusty}
+
+echo "APT::Default-Release \""${VERSION_CODENAME}"\";" > /etc/apt/apt.conf.d/default-distro
sub_source_file=/etc/apt/sources.list.d/yardstick.list
touch "${sub_source_file}"
# first add xenial repo needed for installing qemu_static_user/xenial in the container
# then add complementary architecture repositories in case the cloud image is of different arch
-echo -e "deb [arch="${DOCKER_ARCH}"] "${DOCKER_REPO}" xenial-updates universe
-deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty main universe multiverse restricted
-deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-updates main universe multiverse restricted
-deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-security main universe multiverse restricted
-deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-proposed main universe multiverse restricted" > "${sub_source_file}"
+if [[ "${VERSION_CODENAME}" != "xenial" ]]; then
+ REPO_UPDATE="deb [arch="${DOCKER_ARCH}"] "${DOCKER_REPO}" xenial-updates universe"
+fi
+
+echo -e ""${REPO_UPDATE}"
+deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" "${VERSION_CODENAME}" main universe multiverse restricted
+deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" "${VERSION_CODENAME}"-updates main universe multiverse restricted
+deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" "${VERSION_CODENAME}"-security main universe multiverse restricted
+deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" "${VERSION_CODENAME}"-proposed main universe multiverse restricted" > "${sub_source_file}"
echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf
diff --git a/nsb_setup.sh b/nsb_setup.sh
index 05d1fd6db..88027d9bd 100755
--- a/nsb_setup.sh
+++ b/nsb_setup.sh
@@ -40,7 +40,7 @@ install_libs()
{
echo "Install libs needed to build and run NSB Testing..."
apt-get update > /dev/null 2>&1
- pkg=(git build-essential python-dev virtualenv python-virtualenv virtualenv linux-headers-$(uname -r) unzip python-pip libpcap-dev)
+ pkg=(git build-essential python-dev virtualenv python-virtualenv virtualenv linux-headers-$(uname -r) unzip python-pip libpcap-dev cmake)
for i in "${pkg[@]}"; do
dpkg-query -W --showformat='${Status}\n' "${i}"|grep "install ok installed"
if [ "$?" -eq "1" ]; then
diff --git a/requirements.txt b/requirements.txt
index 2bcc4dfa7..d03c4377f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -38,6 +38,7 @@ iso8601==0.1.11
jsonpatch==1.15
jsonpointer==1.10
jsonschema==2.5.1
+jinja2schema==0.1.4
keystoneauth1==2.18.0
kubernetes==3.0.0a1
linecache2==1.0.0
diff --git a/samples/spec_cpu.yaml b/samples/spec_cpu.yaml
new file mode 100644
index 000000000..227c3e9d2
--- /dev/null
+++ b/samples/spec_cpu.yaml
@@ -0,0 +1,44 @@
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+schema: "yardstick:task:0.1"
+
+{% set file = file or "/etc/yardstick/pod.yaml" %}
+
+scenarios:
+-
+ type: SpecCPU2006
+
+ options:
+ benchmark_subset: int
+ #runspec_tune: all
+ #output_format: all
+ #SPECint_benchmark: perlbench
+ #SPECfp_benchmark: bwaves
+ #runspec_config: yardstick_spec_cpu2006.cfg
+ #runspec_iterations: 1
+ #runspec_tune: base
+ #runspec_size: test
+
+ host: node1.spec_cpu2006
+
+ runner:
+ type: Iteration
+ iterations: 1
+
+context:
+ type: Node
+ name: spec_cpu2006
+ file: {{ file }}
+
+ env:
+ type: ansible
+ setup: spec_cpu2006_install.yaml
+ teardown: spec_cpu2006_uninstall.yaml
diff --git a/tests/unit/apiserver/resources/test_env_action.py b/tests/unit/apiserver/resources/test_env_action.py
index d61092dbc..31afa4862 100644
--- a/tests/unit/apiserver/resources/test_env_action.py
+++ b/tests/unit/apiserver/resources/test_env_action.py
@@ -21,13 +21,13 @@ class EnvTestCase(APITestCase):
data = {'action': 'create_grafana'}
resp = self._post(url, data)
- time.sleep(1)
+ time.sleep(0)
task_id = resp['result']['task_id']
url = '/yardstick/asynctask?task_id={}'.format(task_id)
resp = self._get(url)
- time.sleep(2)
+ time.sleep(0)
self.assertTrue(u'status' in resp)
diff --git a/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py b/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py
new file mode 100644
index 000000000..40423b9da
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.compute.spec_cpu.SpecCPU
+
+from __future__ import absolute_import
+
+import unittest
+
+import mock
+
+from yardstick.common import utils
+from yardstick.benchmark.scenarios.compute import spec_cpu
+
+
+@mock.patch('yardstick.benchmark.scenarios.compute.spec_cpu.ssh')
+class SpecCPUTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.ctx = {
+ 'host': {
+ 'ip': '172.16.0.137',
+ 'user': 'root',
+ 'key_filename': "mykey.key"
+ }
+ }
+
+ self.result = {}
+
+ def test_spec_cpu_successful_setup(self, mock_ssh):
+
+ options = {
+ "SPECint_benchmark": "perlbench",
+ "runspec_tune": "all",
+ "output_format": "all",
+ "runspec_iterations": "1",
+ "runspec_tune": "base",
+ "runspec_size": "test"
+ }
+ args = {"options": options}
+ s = spec_cpu.SpecCPU(args, self.ctx)
+ mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+ s.setup()
+ self.assertIsNotNone(s.client)
+ self.assertTrue(s.setup_done, True)
+
+ def test_spec_cpu_successful__run_no_sla(self, mock_ssh):
+
+ options = {
+ "SPECint_benchmark": "perlbench",
+ "runspec_tune": "all",
+ "output_format": "all"
+ }
+ args = {"options": options}
+ s = spec_cpu.SpecCPU(args, self.ctx)
+
+ sample_output = ''
+ mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+ s.run(self.result)
+ expected_result = {}
+ self.assertEqual(self.result, expected_result)
+
+ def test_ramspeed_unsuccessful_script_error(self, mock_ssh):
+ options = {
+ "benchmark_subset": "int"
+ }
+ args = {"options": options}
+ s = spec_cpu.SpecCPU(args, self.ctx)
+
+ mock_ssh.SSH.from_node().execute.return_value = (1, '', 'FOOBAR')
+ self.assertRaises(RuntimeError, s.run, self.result)
+
+def main():
+ unittest.main()
+
+if __name__ == '__main__':
+ main()
diff --git a/yardstick/benchmark/contexts/model.py b/yardstick/benchmark/contexts/model.py
index aed1a3f60..2db96bade 100644
--- a/yardstick/benchmark/contexts/model.py
+++ b/yardstick/benchmark/contexts/model.py
@@ -257,10 +257,11 @@ class Server(Object): # pragma: no cover
port_name = server_name + "-" + network.name + "-port"
self.ports[network.name] = {"stack_name": port_name}
# we can't use secgroups if port_security_enabled is False
- if network.port_security_enabled:
- sec_group_id = self.secgroup_name
- else:
+ if network.port_security_enabled is False:
sec_group_id = None
+ else:
+ # if port_security_enabled is None we still need to add to secgroup
+ sec_group_id = self.secgroup_name
# don't refactor to pass in network object, that causes JSON
# circular ref encode errors
template.add_port(port_name, network.stack_name, network.subnet_stack_name,
diff --git a/yardstick/benchmark/core/plugin.py b/yardstick/benchmark/core/plugin.py
index c8d0865d1..a741d5e74 100644
--- a/yardstick/benchmark/core/plugin.py
+++ b/yardstick/benchmark/core/plugin.py
@@ -153,7 +153,7 @@ class PluginParser(object):
raise e
print("Input plugin is:\n%s\n" % rendered_plugin)
- cfg = yaml.load(rendered_plugin)
+ cfg = yaml.safe_load(rendered_plugin)
except IOError as ioerror:
sys.exit(ioerror)
diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py
index b2da7a2ee..af508496f 100644
--- a/yardstick/benchmark/core/task.py
+++ b/yardstick/benchmark/core/task.py
@@ -411,7 +411,7 @@ class TaskParser(object): # pragma: no cover
try:
with open(self.path) as stream:
- cfg = yaml.load(stream)
+ cfg = yaml.safe_load(stream)
except IOError as ioerror:
sys.exit(ioerror)
@@ -475,7 +475,7 @@ class TaskParser(object): # pragma: no cover
raise e
print("Input task is:\n%s\n" % rendered_task)
- cfg = yaml.load(rendered_task)
+ cfg = yaml.safe_load(rendered_task)
except IOError as ioerror:
sys.exit(ioerror)
diff --git a/yardstick/benchmark/core/testcase.py b/yardstick/benchmark/core/testcase.py
index 7b23b73aa..7ab1b08cf 100644
--- a/yardstick/benchmark/core/testcase.py
+++ b/yardstick/benchmark/core/testcase.py
@@ -69,7 +69,7 @@ class Testcase(object):
def _parse_testcase(self, testcase_info):
rendered_testcase = TaskTemplate.render(testcase_info)
- testcase_cfg = yaml.load(rendered_testcase)
+ testcase_cfg = yaml.safe_load(rendered_testcase)
test_precondition = testcase_cfg.get('precondition', {})
installer_type = test_precondition.get('installer_type', 'all')
diff --git a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
index 7b3d8b0be..a20b26396 100644
--- a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
+++ b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
@@ -56,7 +56,7 @@ class BaseAttacker(object):
def __init__(self, config, context):
if not BaseAttacker.attacker_cfgs:
with open(attacker_conf_path) as stream:
- BaseAttacker.attacker_cfgs = yaml.load(stream)
+ BaseAttacker.attacker_cfgs = yaml.safe_load(stream)
self._config = config
self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
index ba3370003..6165aba74 100644
--- a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
+++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
@@ -74,7 +74,7 @@ class BaseMonitor(multiprocessing.Process):
def __init__(self, config, context, data):
if not BaseMonitor.monitor_cfgs:
with open(monitor_conf_path) as stream:
- BaseMonitor.monitor_cfgs = yaml.load(stream)
+ BaseMonitor.monitor_cfgs = yaml.safe_load(stream)
multiprocessing.Process.__init__(self)
self._config = config
self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
index 88ca9e2bb..4c2ce82d9 100644
--- a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
+++ b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
@@ -54,7 +54,7 @@ class BaseOperation(object):
def __init__(self, config, context):
if not BaseOperation.operation_cfgs:
with open(operation_conf_path) as stream:
- BaseOperation.operation_cfgs = yaml.load(stream)
+ BaseOperation.operation_cfgs = yaml.safe_load(stream)
self.key = ''
self._config = config
self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
index 1ccd05844..ce34d8be0 100644
--- a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
+++ b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
@@ -58,7 +58,7 @@ class BaseResultChecker(object):
def __init__(self, config, context):
if not BaseResultChecker.resultchecker_cfgs:
with open(resultchecker_conf_path) as stream:
- BaseResultChecker.resultchecker_cfgs = yaml.load(stream)
+ BaseResultChecker.resultchecker_cfgs = yaml.safe_load(stream)
self.actualResult = object()
self.expectedResult = object()
self.success = False
diff --git a/yardstick/benchmark/scenarios/compute/spec_cpu.py b/yardstick/benchmark/scenarios/compute/spec_cpu.py
new file mode 100644
index 000000000..520618631
--- /dev/null
+++ b/yardstick/benchmark/scenarios/compute/spec_cpu.py
@@ -0,0 +1,141 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from __future__ import absolute_import
+
+import logging
+import pkg_resources
+
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class SpecCPU(base.Scenario):
+ """Spec CPU2006 benchmark
+
+ Parameters
+ benchmark_subset - Specifies a subset of SPEC CPU2006 benchmarks to run
+ type: string
+ unit: na
+ default: na
+
+ SPECint_benchmark - A SPECint benchmark to run
+ type: string
+ unit: na
+ default: na
+
+ SPECint_benchmark - A SPECfp benchmark to run
+ type: string
+ unit: na
+ default: na
+
+ output_format - Desired report format
+ type: string
+ unit: na
+ default: na
+
+ runspec_config - SPEC CPU2006 config file provided to the runspec binary
+ type: string
+ unit: na
+ default: "Example-linux64-amd64-gcc43+.cfg"
+
+ runspec_iterations - The number of benchmark iterations to execute.
+ For a reportable run, must be 3.
+ type: int
+ unit: na
+ default: na
+
+ runspec_tune - Tuning to use (base, peak, or all). For a reportable run, must be either
+ base or all. Reportable runs do base first, then (optionally) peak.
+ type: string
+ unit: na
+ default: na
+
+ runspec_size - Size of input data to run (test, train, or ref). Reportable runs ensure
+ that your binaries can produce correct results with the test and train
+ workloads.
+ type: string
+ unit: na
+ default: na
+ """
+ __scenario_type__ = "SpecCPU2006"
+
+ def __init__(self, scenario_cfg, context_cfg):
+ self.scenario_cfg = scenario_cfg
+ self.context_cfg = context_cfg
+ self.setup_done = False
+ self.options = self.scenario_cfg['options']
+
+ def setup(self):
+ """scenario setup"""
+ host = self.context_cfg['host']
+ LOG.info("user:%s, host:%s", host['user'], host['ip'])
+ self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
+ self.client.wait(timeout=600)
+
+ if "runspec_config" in self.options:
+ self.runspec_config = self.options["runspec_config"]
+
+ self.runspec_config_file = pkg_resources.resource_filename(
+ "yardstick.resources", 'files/' + self.runspec_config)
+
+ # copy SPEC CPU2006 config file to host if given
+ self.client._put_file_shell(self.runspec_config_file,
+ '/usr/cpu2006/config/yardstick_spec_cpu2006.cfg')
+ else:
+ self.runspec_config = "Example-linux64-amd64-gcc43+.cfg"
+
+ self.setup_done = True
+
+ def run(self, result):
+ """execute the benchmark"""
+
+ if not self.setup_done:
+ self.setup()
+
+ cmd = "cd /usr/cpu2006/ && . ./shrc && runspec --config %s" % self.runspec_config
+ cmd_args = ""
+
+ if "output_format" in self.options:
+ cmd_args += " --output_format %s" % self.options["output_format"]
+
+ if "runspec_tune" in self.options:
+ cmd_args += " --tune %s" % self.options["runspec_tune"]
+
+ benchmark_subset = self.options.get('benchmark_subset', None)
+ specint_benchmark = self.options.get('SPECint_benchmark', None)
+ specfp_benchmark = self.options.get('SPECfp_benchmark', None)
+
+ if benchmark_subset:
+ cmd_args += " %s" % benchmark_subset
+ else:
+ cmd_args += " --noreportable"
+
+ if "runspec_iterations" in self.options:
+ cmd_args += " --iterations %s" % self.options["runspec_iterations"]
+
+ if "runspec_size" in self.options:
+ cmd_args += " --size %s" % self.options["runspec_size"]
+
+ if specint_benchmark:
+ cmd_args += " %s" % specint_benchmark
+
+ if specfp_benchmark:
+ cmd_args += " %s" % specfp_benchmark
+
+ cmd += "%s" % cmd_args
+
+ LOG.debug("Executing command: %s", cmd)
+ status, stdout, stderr = self.client.execute(cmd, timeout=86400)
+ if status:
+ raise RuntimeError(stderr)
+
+ LOG.info('SPEC CPU2006 benchmark completed, please find benchmark reports \
+ at /tmp/result directory')
diff --git a/yardstick/common/template_format.py b/yardstick/common/template_format.py
index e1662ced1..98c0a0b3c 100644
--- a/yardstick/common/template_format.py
+++ b/yardstick/common/template_format.py
@@ -51,6 +51,7 @@ def parse(tmpl_str):
tpl = jsonutils.loads(tmpl_str)
else:
try:
+ # we already use SafeLoader when constructing special Heat YAML loader class
tpl = yaml.load(tmpl_str, Loader=yaml_loader)
except yaml.YAMLError as yea:
raise ValueError(yea)