diff options
26 files changed, 576 insertions, 22 deletions
diff --git a/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml new file mode 100644 index 000000000..7f72e1394 --- /dev/null +++ b/ansible/roles/archive_spec_cpu2006_result/tasks/main.yaml @@ -0,0 +1,15 @@ +--- +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: archive_spec_cpu2006_result + archive: + path: /usr/cpu2006/result + dest: /usr/cpu2006/spec_cpu2006_result.zip + become: true diff --git a/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml new file mode 100644 index 000000000..370ec7a0c --- /dev/null +++ b/ansible/roles/fetch_spec_cpu2006_result/tasks/main.yaml @@ -0,0 +1,16 @@ +--- +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: fetch_spec_cpu2006_result + fetch: + src: /usr/cpu2006/spec_cpu2006_result.zip + dest: /tmp/ + flat: yes + become: true diff --git a/ansible/roles/install_spec_cpu2006/tasks/main.yaml b/ansible/roles/install_spec_cpu2006/tasks/main.yaml new file mode 100644 index 000000000..baa9f6ca0 --- /dev/null +++ b/ansible/roles/install_spec_cpu2006/tasks/main.yaml @@ -0,0 +1,24 @@ +--- +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: upload SPEC CPU2006 ISO + copy: + src: /home/opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso + dest: ~/cpu2006-1.2.iso + become: true + +- name: install SPEC CPU2006 + shell: + mount -t iso9660 -o ro,exec ~/cpu2006-1.2.iso /mnt; + cd /mnt; + ./install.sh -fd /usr/cpu2006; + args: + executable: /bin/bash + become: true diff --git a/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml b/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml new file mode 100644 index 000000000..86dcc1aeb --- /dev/null +++ b/ansible/roles/unarchive_spec_cpu2006_result/tasks/main.yaml @@ -0,0 +1,16 @@ +--- +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: unarchive_spec_cpu2006_result + unarchive: + src: /tmp/spec_cpu2006_result.zip + dest: /tmp/ + remote_src: True + become: true diff --git a/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml b/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml new file mode 100644 index 000000000..72d8c93ee --- /dev/null +++ b/ansible/roles/uninstall_spec_cpu2006/tasks/main.yaml @@ -0,0 +1,28 @@ +--- +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: remove SPEC CPU 2006 + file: + path: /usr/cpu2006 + state: absent + become: true + +- name: umount SPEC CPU 2006 ISO + shell: + umount -l /mnt; + args: + executable: /bin/bash + become: true + +- name: remove SPEC CPU 2006 ISO + file: + path: ~/cpu2006-1.2.iso + state: absent + become: true diff --git a/ansible/spec_cpu2006_install.yaml b/ansible/spec_cpu2006_install.yaml new file mode 100644 index 000000000..470f5c8d8 --- /dev/null +++ b/ansible/spec_cpu2006_install.yaml @@ -0,0 +1,13 @@ +--- +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: spec_target + roles: + - install_spec_cpu2006 diff --git a/ansible/spec_cpu2006_uninstall.yaml b/ansible/spec_cpu2006_uninstall.yaml new file mode 100644 index 000000000..4991c98b0 --- /dev/null +++ b/ansible/spec_cpu2006_uninstall.yaml @@ -0,0 +1,22 @@ +--- +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: spec_target + roles: + - archive_spec_cpu2006_result + - fetch_spec_cpu2006_result + +- hosts: localhost + roles: + - unarchive_spec_cpu2006_result + +- hosts: spec_target + roles: + - uninstall_spec_cpu2006 diff --git a/api/resources/v1/env.py b/api/resources/v1/env.py index 8943db3d1..8367fa9eb 100644 --- a/api/resources/v1/env.py +++ b/api/resources/v1/env.py @@ -393,7 +393,7 @@ class V1Env(ApiResource): return result_handler(consts.API_ERROR, 'file must be provided') LOG.info('Checking file') - data = yaml.load(pod_file.read()) + data = yaml.safe_load(pod_file.read()) if not isinstance(data, collections.Mapping): return result_handler(consts.API_ERROR, 'invalid yaml file') diff --git a/api/resources/v2/testcases.py b/api/resources/v2/testcases.py index b47a8f6b7..316ef2664 100644 --- a/api/resources/v2/testcases.py +++ b/api/resources/v2/testcases.py @@ -10,6 +10,8 @@ import logging import errno import os +import jinja2schema + from api import ApiResource from yardstick.common.utils import result_handler from yardstick.common import constants as consts @@ -56,7 +58,10 @@ class V2Testcase(ApiResource): if e.errno == errno.ENOENT: return result_handler(consts.API_ERROR, 'case does not exist') - return result_handler(consts.API_SUCCESS, {'testcase': data}) + options = {k: {'description': '', 'type': v.__class__.__name__} + for k, v in jinja2schema.infer(data).items()} + + return result_handler(consts.API_SUCCESS, {'testcase': data, 'args': options}) def delete(self, case_name): case_path = os.path.join(consts.TESTCASE_DIR, '{}.yaml'.format(case_name)) diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc078.rst b/docs/testing/user/userguide/opnfv_yardstick_tc078.rst new file mode 100644 index 000000000..560d55e07 --- /dev/null +++ b/docs/testing/user/userguide/opnfv_yardstick_tc078.rst @@ -0,0 +1,133 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Huawei Technologies Co.,Ltd and others. + +************************************* +Yardstick Test Case Description TC078 +************************************* + +.. _spec_cpu2006: https://www.spec.org/cpu2006/ + ++-----------------------------------------------------------------------------+ +|Compute Performance | +| | ++--------------+--------------------------------------------------------------+ +|test case id | OPNFV_YARDSTICK_TC078_SPEC CPU 2006 | +| | | ++--------------+--------------------------------------------------------------+ +|metric | compute-intensive performance | +| | | ++--------------+--------------------------------------------------------------+ +|test purpose | The purpose of TC078 is to evaluate the IaaS compute | +| | performance by using SPEC CPU 2006 benchmark. The SPEC CPU | +| | 2006 benchmark has several different ways to measure | +| | computer performance. One way is to measure how fast the | +| | computer completes a single task; this is called a speed | +| | measurement. Another way is to measure how many tasks | +| | computer can accomplish in a certain amount of time; this is | +| | called a throughput, capacity or rate measurement. | +| | | ++--------------+--------------------------------------------------------------+ +|test tool | SPEC CPU 2006 | +| | | +| | The SPEC CPU 2006 benchmark is SPEC's industry-standardized, | +| | CPU-intensive benchmark suite, stressing a system's | +| | processor, memory subsystem and compiler. This benchmark | +| | suite includes the SPECint benchmarks and the SPECfp | +| | benchmarks. The SPECint 2006 benchmark contains 12 different | +| | enchmark tests and the SPECfp 2006 benchmark contains 19 | +| | different benchmark tests. | +| | | +| | SPEC CPU 2006 is not always part of a Linux distribution. | +| | SPEC requires that users purchase a license and agree with | +| | their terms and conditions. For this test case, users must | +| | manually download cpu2006-1.2.iso from the SPEC website and | +| | save it under the yardstick/resources folder (e.g. /home/ | +| | opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso) | +| | SPEC CPU® 2006 benchmark is available for purchase via the | +| | SPEC order form (https://www.spec.org/order.html). | +| | | ++--------------+--------------------------------------------------------------+ +|test | This test case uses SPEC CPU 2006 benchmark to measure | +|description | compute-intensive performance of hosts. | +| | | ++--------------+--------------------------------------------------------------+ +|configuration | file: spec_cpu.yaml (in the 'samples' directory) | +| | | +| | benchmark_subset is set to int. | +| | | +| | SLA is not available in this test case. | +| | | ++--------------+--------------------------------------------------------------+ +|applicability | Test can be configured with different: | +| | | +| | * benchmark_subset - a subset of SPEC CPU2006 benchmarks to | +| | run; | +| | * SPECint_benchmark - a SPECint benchmark to run; | +| | * SPECint_benchmark - a SPECfp benchmark to run; | +| | * output_format - desired report format; | +| | * runspec_config - SPEC CPU2006 config file provided to the | +| | runspec binary; | +| | * runspec_iterations - the number of benchmark iterations | +| | to execute. For a reportable run, must be 3; | +| | * runspec_tune - tuning to use (base, peak, or all). For a | +| | reportable run, must be either base or all. Reportable | +| | runs do base first, then (optionally) peak; | +| | * runspec_size - size of input data to run (test, train, or | +| | ref). Reportable runs ensure that your binaries can | +| | produce correct results with the test and train workloads | +| | | ++--------------+--------------------------------------------------------------+ +|usability | This test case is used for executing SPEC CPU 2006 benchmark | +| | physical servers. The SPECint 2006 benchmark takes | +| | approximately 5 hours. | +| | | ++--------------+--------------------------------------------------------------+ +|references | spec_cpu2006_ | +| | | +| | ETSI-NFV-TST001 | +| | | ++--------------+--------------------------------------------------------------+ +|pre-test | To run and install SPEC CPU2006, the following are required: | +|conditions | * For SPECint2006: Both C99 and C++98 compilers; | +| | * For SPECfp2006: All three of C99, C++98 and Fortran-95 | +| | compilers; | +| | * At least 8GB of disk space availabile on the system. | +| | | ++--------------+--------------------------------------------------------------+ +|test sequence | description and expected result | +| | | ++--------------+--------------------------------------------------------------+ +|step 1 | cpu2006-1.2.iso has been saved under the yardstick/resources | +| | folder (e.g. /home/opnfv/repos/yardstick/yardstick/resources | +| | /cpu2006-1.2.iso). Additional, to use your custom runspec | +| | config file you can save it under the yardstick/resources/ | +| | files folder and specify the config file name in the | +| | runspec_config parameter. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Upload SPEC CPU2006 ISO to the target server and install | +| | SPEC CPU2006 via ansible. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | Yardstick is connected with the target server by using ssh. | +| | If custom runspec config file is used, this file is copyied | +| | from yardstick to the target server via the ssh tunnel. | +--------------+---------------------------------------------------------------+ +|step 4 | SPEC CPU2006 benchmark is invoked and SPEC CPU 2006 metrics | +| | are generated. | +| | | ++--------------+--------------------------------------------------------------+ +|step 5 | Text, HTML, CSV, PDF, and Configuration file outputs for the | +| | SPEC CPU 2006 metrics are fetch from the server and stored | +| | under /tmp/result folder. | +| | | ++--------------+--------------------------------------------------------------+ +|step 6 | uninstall SPEC CPU2006 and remove cpu2006-1.2.iso from the | +| | target server . | +| | | ++--------------+--------------------------------------------------------------+ +|test verdict | None. SPEC CPU2006 results are collected and stored. | +| | | ++--------------+--------------------------------------------------------------+ diff --git a/install.sh b/install.sh index 8a5050a61..4ca72a757 100755 --- a/install.sh +++ b/install.sh @@ -14,6 +14,7 @@ DOCKER_ARCH="$(uname -m)" UBUNTU_PORTS_URL="http://ports.ubuntu.com/ubuntu-ports/" UBUNTU_ARCHIVE_URL="http://archive.ubuntu.com/ubuntu/" +source /etc/os-release source_file=/etc/apt/sources.list if [[ "${DOCKER_ARCH}" == "aarch64" ]]; then @@ -33,18 +34,25 @@ else fi sed -i -e 's/^deb-src /# deb-src /g' "${source_file}" -echo "APT::Default-Release \"trusty\";" > /etc/apt/apt.conf.d/default-distro + +VERSION_CODENAME=${VERSION_CODENAME:-trusty} + +echo "APT::Default-Release \""${VERSION_CODENAME}"\";" > /etc/apt/apt.conf.d/default-distro sub_source_file=/etc/apt/sources.list.d/yardstick.list touch "${sub_source_file}" # first add xenial repo needed for installing qemu_static_user/xenial in the container # then add complementary architecture repositories in case the cloud image is of different arch -echo -e "deb [arch="${DOCKER_ARCH}"] "${DOCKER_REPO}" xenial-updates universe -deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty main universe multiverse restricted -deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-updates main universe multiverse restricted -deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-security main universe multiverse restricted -deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-proposed main universe multiverse restricted" > "${sub_source_file}" +if [[ "${VERSION_CODENAME}" != "xenial" ]]; then + REPO_UPDATE="deb [arch="${DOCKER_ARCH}"] "${DOCKER_REPO}" xenial-updates universe" +fi + +echo -e ""${REPO_UPDATE}" +deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" "${VERSION_CODENAME}" main universe multiverse restricted +deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" "${VERSION_CODENAME}"-updates main universe multiverse restricted +deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" "${VERSION_CODENAME}"-security main universe multiverse restricted +deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" "${VERSION_CODENAME}"-proposed main universe multiverse restricted" > "${sub_source_file}" echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf diff --git a/nsb_setup.sh b/nsb_setup.sh index 05d1fd6db..88027d9bd 100755 --- a/nsb_setup.sh +++ b/nsb_setup.sh @@ -40,7 +40,7 @@ install_libs() { echo "Install libs needed to build and run NSB Testing..." apt-get update > /dev/null 2>&1 - pkg=(git build-essential python-dev virtualenv python-virtualenv virtualenv linux-headers-$(uname -r) unzip python-pip libpcap-dev) + pkg=(git build-essential python-dev virtualenv python-virtualenv virtualenv linux-headers-$(uname -r) unzip python-pip libpcap-dev cmake) for i in "${pkg[@]}"; do dpkg-query -W --showformat='${Status}\n' "${i}"|grep "install ok installed" if [ "$?" -eq "1" ]; then diff --git a/requirements.txt b/requirements.txt index 2bcc4dfa7..d03c4377f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,6 +38,7 @@ iso8601==0.1.11 jsonpatch==1.15 jsonpointer==1.10 jsonschema==2.5.1 +jinja2schema==0.1.4 keystoneauth1==2.18.0 kubernetes==3.0.0a1 linecache2==1.0.0 diff --git a/samples/spec_cpu.yaml b/samples/spec_cpu.yaml new file mode 100644 index 000000000..227c3e9d2 --- /dev/null +++ b/samples/spec_cpu.yaml @@ -0,0 +1,44 @@ +--- +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +schema: "yardstick:task:0.1" + +{% set file = file or "/etc/yardstick/pod.yaml" %} + +scenarios: +- + type: SpecCPU2006 + + options: + benchmark_subset: int + #runspec_tune: all + #output_format: all + #SPECint_benchmark: perlbench + #SPECfp_benchmark: bwaves + #runspec_config: yardstick_spec_cpu2006.cfg + #runspec_iterations: 1 + #runspec_tune: base + #runspec_size: test + + host: node1.spec_cpu2006 + + runner: + type: Iteration + iterations: 1 + +context: + type: Node + name: spec_cpu2006 + file: {{ file }} + + env: + type: ansible + setup: spec_cpu2006_install.yaml + teardown: spec_cpu2006_uninstall.yaml diff --git a/tests/unit/apiserver/resources/test_env_action.py b/tests/unit/apiserver/resources/test_env_action.py index d61092dbc..31afa4862 100644 --- a/tests/unit/apiserver/resources/test_env_action.py +++ b/tests/unit/apiserver/resources/test_env_action.py @@ -21,13 +21,13 @@ class EnvTestCase(APITestCase): data = {'action': 'create_grafana'} resp = self._post(url, data) - time.sleep(1) + time.sleep(0) task_id = resp['result']['task_id'] url = '/yardstick/asynctask?task_id={}'.format(task_id) resp = self._get(url) - time.sleep(2) + time.sleep(0) self.assertTrue(u'status' in resp) diff --git a/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py b/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py new file mode 100644 index 000000000..40423b9da --- /dev/null +++ b/tests/unit/benchmark/scenarios/compute/test_spec_cpu.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Unittest for yardstick.benchmark.scenarios.compute.spec_cpu.SpecCPU + +from __future__ import absolute_import + +import unittest + +import mock + +from yardstick.common import utils +from yardstick.benchmark.scenarios.compute import spec_cpu + + +@mock.patch('yardstick.benchmark.scenarios.compute.spec_cpu.ssh') +class SpecCPUTestCase(unittest.TestCase): + + def setUp(self): + self.ctx = { + 'host': { + 'ip': '172.16.0.137', + 'user': 'root', + 'key_filename': "mykey.key" + } + } + + self.result = {} + + def test_spec_cpu_successful_setup(self, mock_ssh): + + options = { + "SPECint_benchmark": "perlbench", + "runspec_tune": "all", + "output_format": "all", + "runspec_iterations": "1", + "runspec_tune": "base", + "runspec_size": "test" + } + args = {"options": options} + s = spec_cpu.SpecCPU(args, self.ctx) + mock_ssh.SSH.from_node().execute.return_value = (0, '', '') + + s.setup() + self.assertIsNotNone(s.client) + self.assertTrue(s.setup_done, True) + + def test_spec_cpu_successful__run_no_sla(self, mock_ssh): + + options = { + "SPECint_benchmark": "perlbench", + "runspec_tune": "all", + "output_format": "all" + } + args = {"options": options} + s = spec_cpu.SpecCPU(args, self.ctx) + + sample_output = '' + mock_ssh.SSH.from_node().execute.return_value = (0, '', '') + s.run(self.result) + expected_result = {} + self.assertEqual(self.result, expected_result) + + def test_ramspeed_unsuccessful_script_error(self, mock_ssh): + options = { + "benchmark_subset": "int" + } + args = {"options": options} + s = spec_cpu.SpecCPU(args, self.ctx) + + mock_ssh.SSH.from_node().execute.return_value = (1, '', 'FOOBAR') + self.assertRaises(RuntimeError, s.run, self.result) + +def main(): + unittest.main() + +if __name__ == '__main__': + main() diff --git a/yardstick/benchmark/contexts/model.py b/yardstick/benchmark/contexts/model.py index aed1a3f60..2db96bade 100644 --- a/yardstick/benchmark/contexts/model.py +++ b/yardstick/benchmark/contexts/model.py @@ -257,10 +257,11 @@ class Server(Object): # pragma: no cover port_name = server_name + "-" + network.name + "-port" self.ports[network.name] = {"stack_name": port_name} # we can't use secgroups if port_security_enabled is False - if network.port_security_enabled: - sec_group_id = self.secgroup_name - else: + if network.port_security_enabled is False: sec_group_id = None + else: + # if port_security_enabled is None we still need to add to secgroup + sec_group_id = self.secgroup_name # don't refactor to pass in network object, that causes JSON # circular ref encode errors template.add_port(port_name, network.stack_name, network.subnet_stack_name, diff --git a/yardstick/benchmark/core/plugin.py b/yardstick/benchmark/core/plugin.py index c8d0865d1..a741d5e74 100644 --- a/yardstick/benchmark/core/plugin.py +++ b/yardstick/benchmark/core/plugin.py @@ -153,7 +153,7 @@ class PluginParser(object): raise e print("Input plugin is:\n%s\n" % rendered_plugin) - cfg = yaml.load(rendered_plugin) + cfg = yaml.safe_load(rendered_plugin) except IOError as ioerror: sys.exit(ioerror) diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py index b2da7a2ee..af508496f 100644 --- a/yardstick/benchmark/core/task.py +++ b/yardstick/benchmark/core/task.py @@ -411,7 +411,7 @@ class TaskParser(object): # pragma: no cover try: with open(self.path) as stream: - cfg = yaml.load(stream) + cfg = yaml.safe_load(stream) except IOError as ioerror: sys.exit(ioerror) @@ -475,7 +475,7 @@ class TaskParser(object): # pragma: no cover raise e print("Input task is:\n%s\n" % rendered_task) - cfg = yaml.load(rendered_task) + cfg = yaml.safe_load(rendered_task) except IOError as ioerror: sys.exit(ioerror) diff --git a/yardstick/benchmark/core/testcase.py b/yardstick/benchmark/core/testcase.py index 7b23b73aa..7ab1b08cf 100644 --- a/yardstick/benchmark/core/testcase.py +++ b/yardstick/benchmark/core/testcase.py @@ -69,7 +69,7 @@ class Testcase(object): def _parse_testcase(self, testcase_info): rendered_testcase = TaskTemplate.render(testcase_info) - testcase_cfg = yaml.load(rendered_testcase) + testcase_cfg = yaml.safe_load(rendered_testcase) test_precondition = testcase_cfg.get('precondition', {}) installer_type = test_precondition.get('installer_type', 'all') diff --git a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py index 7b3d8b0be..a20b26396 100644 --- a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py +++ b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py @@ -56,7 +56,7 @@ class BaseAttacker(object): def __init__(self, config, context): if not BaseAttacker.attacker_cfgs: with open(attacker_conf_path) as stream: - BaseAttacker.attacker_cfgs = yaml.load(stream) + BaseAttacker.attacker_cfgs = yaml.safe_load(stream) self._config = config self._context = context diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py index ba3370003..6165aba74 100644 --- a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py +++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py @@ -74,7 +74,7 @@ class BaseMonitor(multiprocessing.Process): def __init__(self, config, context, data): if not BaseMonitor.monitor_cfgs: with open(monitor_conf_path) as stream: - BaseMonitor.monitor_cfgs = yaml.load(stream) + BaseMonitor.monitor_cfgs = yaml.safe_load(stream) multiprocessing.Process.__init__(self) self._config = config self._context = context diff --git a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py index 88ca9e2bb..4c2ce82d9 100644 --- a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py +++ b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py @@ -54,7 +54,7 @@ class BaseOperation(object): def __init__(self, config, context): if not BaseOperation.operation_cfgs: with open(operation_conf_path) as stream: - BaseOperation.operation_cfgs = yaml.load(stream) + BaseOperation.operation_cfgs = yaml.safe_load(stream) self.key = '' self._config = config self._context = context diff --git a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py index 1ccd05844..ce34d8be0 100644 --- a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py +++ b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py @@ -58,7 +58,7 @@ class BaseResultChecker(object): def __init__(self, config, context): if not BaseResultChecker.resultchecker_cfgs: with open(resultchecker_conf_path) as stream: - BaseResultChecker.resultchecker_cfgs = yaml.load(stream) + BaseResultChecker.resultchecker_cfgs = yaml.safe_load(stream) self.actualResult = object() self.expectedResult = object() self.success = False diff --git a/yardstick/benchmark/scenarios/compute/spec_cpu.py b/yardstick/benchmark/scenarios/compute/spec_cpu.py new file mode 100644 index 000000000..520618631 --- /dev/null +++ b/yardstick/benchmark/scenarios/compute/spec_cpu.py @@ -0,0 +1,141 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from __future__ import absolute_import + +import logging +import pkg_resources + +import yardstick.ssh as ssh +from yardstick.benchmark.scenarios import base + +LOG = logging.getLogger(__name__) + + +class SpecCPU(base.Scenario): + """Spec CPU2006 benchmark + + Parameters + benchmark_subset - Specifies a subset of SPEC CPU2006 benchmarks to run + type: string + unit: na + default: na + + SPECint_benchmark - A SPECint benchmark to run + type: string + unit: na + default: na + + SPECint_benchmark - A SPECfp benchmark to run + type: string + unit: na + default: na + + output_format - Desired report format + type: string + unit: na + default: na + + runspec_config - SPEC CPU2006 config file provided to the runspec binary + type: string + unit: na + default: "Example-linux64-amd64-gcc43+.cfg" + + runspec_iterations - The number of benchmark iterations to execute. + For a reportable run, must be 3. + type: int + unit: na + default: na + + runspec_tune - Tuning to use (base, peak, or all). For a reportable run, must be either + base or all. Reportable runs do base first, then (optionally) peak. + type: string + unit: na + default: na + + runspec_size - Size of input data to run (test, train, or ref). Reportable runs ensure + that your binaries can produce correct results with the test and train + workloads. + type: string + unit: na + default: na + """ + __scenario_type__ = "SpecCPU2006" + + def __init__(self, scenario_cfg, context_cfg): + self.scenario_cfg = scenario_cfg + self.context_cfg = context_cfg + self.setup_done = False + self.options = self.scenario_cfg['options'] + + def setup(self): + """scenario setup""" + host = self.context_cfg['host'] + LOG.info("user:%s, host:%s", host['user'], host['ip']) + self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"}) + self.client.wait(timeout=600) + + if "runspec_config" in self.options: + self.runspec_config = self.options["runspec_config"] + + self.runspec_config_file = pkg_resources.resource_filename( + "yardstick.resources", 'files/' + self.runspec_config) + + # copy SPEC CPU2006 config file to host if given + self.client._put_file_shell(self.runspec_config_file, + '/usr/cpu2006/config/yardstick_spec_cpu2006.cfg') + else: + self.runspec_config = "Example-linux64-amd64-gcc43+.cfg" + + self.setup_done = True + + def run(self, result): + """execute the benchmark""" + + if not self.setup_done: + self.setup() + + cmd = "cd /usr/cpu2006/ && . ./shrc && runspec --config %s" % self.runspec_config + cmd_args = "" + + if "output_format" in self.options: + cmd_args += " --output_format %s" % self.options["output_format"] + + if "runspec_tune" in self.options: + cmd_args += " --tune %s" % self.options["runspec_tune"] + + benchmark_subset = self.options.get('benchmark_subset', None) + specint_benchmark = self.options.get('SPECint_benchmark', None) + specfp_benchmark = self.options.get('SPECfp_benchmark', None) + + if benchmark_subset: + cmd_args += " %s" % benchmark_subset + else: + cmd_args += " --noreportable" + + if "runspec_iterations" in self.options: + cmd_args += " --iterations %s" % self.options["runspec_iterations"] + + if "runspec_size" in self.options: + cmd_args += " --size %s" % self.options["runspec_size"] + + if specint_benchmark: + cmd_args += " %s" % specint_benchmark + + if specfp_benchmark: + cmd_args += " %s" % specfp_benchmark + + cmd += "%s" % cmd_args + + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.client.execute(cmd, timeout=86400) + if status: + raise RuntimeError(stderr) + + LOG.info('SPEC CPU2006 benchmark completed, please find benchmark reports \ + at /tmp/result directory') diff --git a/yardstick/common/template_format.py b/yardstick/common/template_format.py index e1662ced1..98c0a0b3c 100644 --- a/yardstick/common/template_format.py +++ b/yardstick/common/template_format.py @@ -51,6 +51,7 @@ def parse(tmpl_str): tpl = jsonutils.loads(tmpl_str) else: try: + # we already use SafeLoader when constructing special Heat YAML loader class tpl = yaml.load(tmpl_str, Loader=yaml_loader) except yaml.YAMLError as yea: raise ValueError(yea) |