summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc084.rst140
-rw-r--r--tests/unit/common/__init__.py0
-rw-r--r--tools/coding-checks.sh9
-rwxr-xr-xtools/run_tests.sh8
-rw-r--r--tox.ini2
-rwxr-xr-xyardstick/tests/functional/test_cli_runner.py (renamed from tests/functional/test_cli_runner.py)4
-rwxr-xr-xyardstick/tests/functional/test_cli_scenario.py (renamed from tests/functional/test_cli_scenario.py)4
-rwxr-xr-xyardstick/tests/functional/utils.py (renamed from tests/functional/utils.py)0
-rw-r--r--yardstick/tests/unit/apiserver/__init__.py (renamed from tests/unit/apiserver/__init__.py)17
-rw-r--r--[-rwxr-xr-x]yardstick/tests/unit/apiserver/resources/__init__.py (renamed from tests/functional/__init__.py)0
-rw-r--r--yardstick/tests/unit/apiserver/resources/test_env_action.py (renamed from tests/unit/apiserver/resources/test_env_action.py)3
-rw-r--r--yardstick/tests/unit/apiserver/utils/test_influx.py (renamed from tests/unit/apiserver/utils/test_influx.py)8
-rw-r--r--yardstick/tests/unit/common/__init__.py (renamed from tests/unit/apiserver/resources/__init__.py)0
-rw-r--r--yardstick/tests/unit/common/config_sample.yaml (renamed from tests/unit/common/config_sample.yaml)0
-rw-r--r--yardstick/tests/unit/common/test_ansible_common.py (renamed from tests/unit/common/test_ansible_common.py)12
-rw-r--r--yardstick/tests/unit/common/test_httpClient.py (renamed from tests/unit/common/test_httpClient.py)0
-rw-r--r--yardstick/tests/unit/common/test_openstack_utils.py (renamed from tests/unit/common/test_openstack_utils.py)2
-rw-r--r--yardstick/tests/unit/common/test_process.py (renamed from tests/unit/common/test_process.py)0
-rw-r--r--yardstick/tests/unit/common/test_template_format.py (renamed from tests/unit/common/test_template_format.py)3
-rw-r--r--yardstick/tests/unit/common/test_utils.py (renamed from tests/unit/common/test_utils.py)0
-rw-r--r--yardstick/tests/unit/common/test_yaml_loader.py (renamed from tests/unit/common/test_yaml_loader.py)0
-rw-r--r--yardstick/tests/unit/test_ssh.py (renamed from tests/unit/test_ssh.py)2
22 files changed, 186 insertions, 28 deletions
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc084.rst b/docs/testing/user/userguide/opnfv_yardstick_tc084.rst
new file mode 100644
index 000000000..2e7b28e25
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc084.rst
@@ -0,0 +1,140 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC084
+*************************************
+
+.. _spec_cpu_2006: https://www.spec.org/cpu2006/
+
++-----------------------------------------------------------------------------+
+|Compute Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC084_SPEC CPU 2006 FOR VM |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | compute-intensive performance |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC084 is to evaluate the IaaS compute |
+| | performance by using SPEC CPU 2006 benchmark. The SPEC CPU |
+| | 2006 benchmark has several different ways to measure |
+| | computer performance. One way is to measure how fast the |
+| | computer completes a single task; this is called a speed |
+| | measurement. Another way is to measure how many tasks |
+| | computer can accomplish in a certain amount of time; this is |
+| | called a throughput, capacity or rate measurement. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | SPEC CPU 2006 |
+| | |
+| | The SPEC CPU 2006 benchmark is SPEC's industry-standardized, |
+| | CPU-intensive benchmark suite, stressing a system's |
+| | processor, memory subsystem and compiler. This benchmark |
+| | suite includes the SPECint benchmarks and the SPECfp |
+| | benchmarks. The SPECint 2006 benchmark contains 12 different |
+| | benchmark tests and the SPECfp 2006 benchmark contains 19 |
+| | different benchmark tests. |
+| | |
+| | SPEC CPU 2006 is not always part of a Linux distribution. |
+| | SPEC requires that users purchase a license and agree with |
+| | their terms and conditions. For this test case, users must |
+| | manually download cpu2006-1.2.iso from the SPEC website and |
+| | save it under the yardstick/resources folder (e.g. /home/ |
+| | opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso) |
+| | SPEC CPU® 2006 benchmark is available for purchase via the |
+| | SPEC order form (https://www.spec.org/order.html). |
+| | |
++--------------+--------------------------------------------------------------+
+|test | This test case uses SPEC CPU 2006 benchmark to measure |
+|description | compute-intensive performance of VMs. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc084.yaml |
+| | |
+| | benchmark_subset is set to int. |
+| | |
+| | SLA is not available in this test case. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * benchmark_subset - a subset of SPEC CPU 2006 benchmarks |
+| | to run; |
+| | * SPECint_benchmark - a SPECint benchmark to run; |
+| | * SPECint_benchmark - a SPECfp benchmark to run; |
+| | * output_format - desired report format; |
+| | * runspec_config - SPEC CPU 2006 config file provided to |
+| | the runspec binary; |
+| | * runspec_iterations - the number of benchmark iterations |
+| | to execute. For a reportable run, must be 3; |
+| | * runspec_tune - tuning to use (base, peak, or all). For a |
+| | reportable run, must be either base or all. Reportable |
+| | runs do base first, then (optionally) peak; |
+| | * runspec_size - size of input data to run (test, train, or |
+| | ref). Reportable runs ensure that your binaries can |
+| | produce correct results with the test and train workloads |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is used for executing SPEC CPU 2006 benchmark |
+| | on virtual machines. The SPECint 2006 benchmark takes |
+| | approximately 5 hours. (The time may vary due to different |
+| | VM cpu configurations) |
+| | |
++--------------+--------------------------------------------------------------+
+|references | spec_cpu_2006_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | To run and install SPEC CPU 2006, the following are |
+|conditions | required: |
+| | * For SPECint 2006: Both C99 and C++98 compilers are |
+| | installed in VM images; |
+| | * For SPECfp 2006: All three of C99, C++98 and Fortran-95 |
+| | compilers installed in VM images; |
+| | * At least 4GB of disk space availabile on VM. |
+| | |
+| | gcc 4.8.* and g++ 4.8.* version have been tested in Ubuntu |
+| | 14.04, Ubuntu 16.04 and Redhat Enterprise Linux 7.4 image. |
+| | Higher gcc and g++ version may cause compiling error. |
+| | |
+| | For more SPEC CPU 2006 dependencies please visit |
+| | (https://www.spec.org/cpu2006/Docs/techsupport.html) |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | cpu2006-1.2.iso has been saved under the yardstick/resources |
+| | folder (e.g. /home/opnfv/repos/yardstick/yardstick/resources |
+| | /cpu2006-1.2.iso). Additionally, to use your custom runspec |
+| | config file you can save it under the yardstick/resources/ |
+| | files folder and specify the config file name in the |
+| | runspec_config parameter. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Upload SPEC CPU 2006 ISO to the target VM using scp and |
+| | install SPEC CPU 2006. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | Connect to the target server using SSH. |
+| | If custom runspec config file is used, copy this file from |
+| | yardstick to the target VM via the SSH tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | SPEC CPU 2006 benchmark is invoked and SPEC CPU 2006 metrics |
+| | are generated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 5 | Text, HTML, CSV, PDF, and Configuration file outputs for the |
+| | SPEC CPU 2006 metrics are fetched from the VM and stored |
+| | under /tmp/result folder. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. SPEC CPU 2006 results are collected and stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/tests/unit/common/__init__.py b/tests/unit/common/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/tests/unit/common/__init__.py
+++ /dev/null
diff --git a/tools/coding-checks.sh b/tools/coding-checks.sh
index 4ee909988..615f2c3de 100644
--- a/tools/coding-checks.sh
+++ b/tools/coding-checks.sh
@@ -28,6 +28,7 @@ process_options () {
run_pylint () {
local target="${scriptargs:-all}"
+ local output_format=""
if [ "$target" = "all" ]; then
files="ansible api tests yardstick"
@@ -37,11 +38,15 @@ run_pylint () {
*) echo "$target is an unrecognized basecommit"; exit 1;;
esac
fi
-
+ # make Jenkins output parseable because Jenkins doesn't handle color
+ # enventually we should use the Jenkins Pylint plugin or other tools
+ if [ -n "${BRANCH:-}" ] ; then
+ output_format="--output-format=parseable"
+ fi
echo "Running pylint..."
echo "You can speed this up by running it on 'HEAD~[0-9]' (e.g. HEAD~0, this change only)..."
if [ -n "${files}" ]; then
- pylint --rcfile=.pylintrc ${files}
+ pylint --rcfile=.pylintrc ${output_format} ${files}
else
echo "No python changes in this commit, pylint check not required."
exit 0
diff --git a/tools/run_tests.sh b/tools/run_tests.sh
index ddb5e55af..32c4f19e4 100755
--- a/tools/run_tests.sh
+++ b/tools/run_tests.sh
@@ -29,9 +29,16 @@ run_tests() {
echo "Running unittest ... "
if [ $FILE_OPTION == "f" ]; then
python -m unittest discover -v -s tests/unit > $logfile 2>&1
+ if [ $? -ne 0 ]; then
+ echo "FAILED, results in $logfile"
+ exit 1
+ fi
python -m unittest discover -v -s yardstick/tests/unit >> $logfile 2>&1
else
python -m unittest discover -v -s tests/unit
+ if [ $? -ne 0 ]; then
+ exit 1
+ fi
python -m unittest discover -v -s yardstick/tests/unit
fi
@@ -56,7 +63,6 @@ run_functional_test() {
mkdir -p .testrepository
python -m subunit.run discover yardstick/tests/functional > .testrepository/subunit.log
- python -m subunit.run discover tests/functional >> .testrepository/subunit.log
subunit2pyunit < .testrepository/subunit.log
EXIT_CODE=$?
diff --git a/tox.ini b/tox.ini
index 9a82878b0..822ffdab4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,7 @@ envlist = py{27,3},pep8,functional{,-py3},coverage
[testenv]
usedevelop=True
-passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY BRANCH
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
diff --git a/tests/functional/test_cli_runner.py b/yardstick/tests/functional/test_cli_runner.py
index 620edc396..2f2d7fef2 100755
--- a/tests/functional/test_cli_runner.py
+++ b/yardstick/tests/functional/test_cli_runner.py
@@ -7,11 +7,9 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-
-from __future__ import absolute_import
import unittest
-from tests.functional import utils
+from yardstick.tests.functional import utils
class RunnerTestCase(unittest.TestCase):
diff --git a/tests/functional/test_cli_scenario.py b/yardstick/tests/functional/test_cli_scenario.py
index 63b533b85..7aaacad0a 100755
--- a/tests/functional/test_cli_scenario.py
+++ b/yardstick/tests/functional/test_cli_scenario.py
@@ -7,11 +7,9 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-
-from __future__ import absolute_import
import unittest
-from tests.functional import utils
+from yardstick.tests.functional import utils
class ScenarioTestCase(unittest.TestCase):
diff --git a/tests/functional/utils.py b/yardstick/tests/functional/utils.py
index d889c0dfa..d889c0dfa 100755
--- a/tests/functional/utils.py
+++ b/yardstick/tests/functional/utils.py
diff --git a/tests/unit/apiserver/__init__.py b/yardstick/tests/unit/apiserver/__init__.py
index 5e1ed2ea1..44d163429 100644
--- a/tests/unit/apiserver/__init__.py
+++ b/yardstick/tests/unit/apiserver/__init__.py
@@ -1,3 +1,12 @@
+##############################################################################
+# Copyright (c) 2017
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+"""Tests for yardstick/api/server.py"""
from __future__ import absolute_import
import mock
@@ -12,14 +21,16 @@ from yardstick.common import constants as consts
class APITestCase(unittest.TestCase):
-
+ """Tests for the YardStick API server"""
def setUp(self):
self.db_fd, self.db_path = tempfile.mkstemp()
consts.SQLITE = 'sqlite:///{}'.format(self.db_path)
- # server calls gethostbyname which takes 4 seconds, and we should mock it anyway
+ # server calls gethostbyname which takes 4 seconds, and we should mock
+ # it anyway
self.socket_mock = mock.patch.dict("sys.modules", {"socket": mock.MagicMock(
- **{"gethostbyname.return_value": "127.0.0.1", "gethostname.return_value": "localhost"})})
+ **{"gethostbyname.return_value": "127.0.0.1",
+ "gethostname.return_value": "localhost"})})
self.socket_mock.start()
try:
from api import server
diff --git a/tests/functional/__init__.py b/yardstick/tests/unit/apiserver/resources/__init__.py
index e69de29bb..e69de29bb 100755..100644
--- a/tests/functional/__init__.py
+++ b/yardstick/tests/unit/apiserver/resources/__init__.py
diff --git a/tests/unit/apiserver/resources/test_env_action.py b/yardstick/tests/unit/apiserver/resources/test_env_action.py
index 5417ad953..b7bfe294d 100644
--- a/tests/unit/apiserver/resources/test_env_action.py
+++ b/yardstick/tests/unit/apiserver/resources/test_env_action.py
@@ -6,12 +6,11 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from __future__ import absolute_import
import time
import unittest
-from tests.unit.apiserver import APITestCase
+from yardstick.tests.unit.apiserver import APITestCase
class EnvTestCase(APITestCase):
diff --git a/tests/unit/apiserver/utils/test_influx.py b/yardstick/tests/unit/apiserver/utils/test_influx.py
index aff0cab5c..883608bb2 100644
--- a/tests/unit/apiserver/utils/test_influx.py
+++ b/yardstick/tests/unit/apiserver/utils/test_influx.py
@@ -6,13 +6,11 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from __future__ import absolute_import
import unittest
import mock
from api.utils import influx
-
-import six.moves.configparser as ConfigParser
+from six.moves import configparser as ConfigParser
class GetDataDbClientTestCase(unittest.TestCase):
@@ -25,7 +23,7 @@ class GetDataDbClientTestCase(unittest.TestCase):
mock_parser.NoOptionError = ConfigParser.NoOptionError
try:
influx.get_data_db_client()
- except Exception as e:
+ except Exception as e: # pylint: disable=broad-except
self.assertIsInstance(e, RuntimeError)
@@ -50,7 +48,7 @@ class QueryTestCase(unittest.TestCase):
try:
sql = 'select * form tasklist'
influx.query(sql)
- except Exception as e:
+ except Exception as e: # pylint: disable=broad-except
self.assertIsInstance(e, RuntimeError)
diff --git a/tests/unit/apiserver/resources/__init__.py b/yardstick/tests/unit/common/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/unit/apiserver/resources/__init__.py
+++ b/yardstick/tests/unit/common/__init__.py
diff --git a/tests/unit/common/config_sample.yaml b/yardstick/tests/unit/common/config_sample.yaml
index 09218cc79..09218cc79 100644
--- a/tests/unit/common/config_sample.yaml
+++ b/yardstick/tests/unit/common/config_sample.yaml
diff --git a/tests/unit/common/test_ansible_common.py b/yardstick/tests/unit/common/test_ansible_common.py
index 1ef8eee5f..89ea128af 100644
--- a/tests/unit/common/test_ansible_common.py
+++ b/yardstick/tests/unit/common/test_ansible_common.py
@@ -49,7 +49,7 @@ class OverwriteDictTestCase(unittest.TestCase):
class FilenameGeneratorTestCase(unittest.TestCase):
@mock.patch('{}.NamedTemporaryFile'.format(PREFIX))
- def test__handle_existing_file(self, mock_tmp):
+ def test__handle_existing_file(self, _):
ansible_common.FileNameGenerator._handle_existing_file("/dev/null")
def test_get_generator_from_file(self):
@@ -184,7 +184,7 @@ class AnsibleCommonTestCase(unittest.TestCase):
self.assertEqual(a.deploy_dir, "d")
@mock.patch('{}.open'.format(PREFIX))
- def test__gen_ansible_playbook_file_list(self, mock_open):
+ def test__gen_ansible_playbook_file_list(self, _):
d = tempfile.mkdtemp()
try:
a = ansible_common.AnsibleCommon({})
@@ -194,7 +194,7 @@ class AnsibleCommonTestCase(unittest.TestCase):
@mock.patch('{}.NamedTemporaryFile'.format(PREFIX))
@mock.patch('{}.open'.format(PREFIX))
- def test__gen_ansible_inventory_file(self, mock_open, mock_tmp):
+ def test__gen_ansible_inventory_file(self, _, __):
nodes = [{
"name": "name", "user": "user", "password": "PASS",
"role": "role",
@@ -213,7 +213,7 @@ class AnsibleCommonTestCase(unittest.TestCase):
@mock.patch('{}.NamedTemporaryFile'.format(PREFIX))
@mock.patch('{}.open'.format(PREFIX))
- def test__gen_ansible_playbook_file_list_multiple(self, mock_open, mock_tmp):
+ def test__gen_ansible_playbook_file_list_multiple(self, _, __):
d = tempfile.mkdtemp()
try:
a = ansible_common.AnsibleCommon({})
@@ -224,7 +224,7 @@ class AnsibleCommonTestCase(unittest.TestCase):
@mock.patch('{}.NamedTemporaryFile'.format(PREFIX))
@mock.patch('{}.Popen'.format(PREFIX))
@mock.patch('{}.open'.format(PREFIX))
- def test_do_install_tmp_dir(self, mock_open, mock_popen, mock_tmp):
+ def test_do_install_tmp_dir(self, _, mock_popen, __):
mock_popen.return_value.communicate.return_value = "", ""
mock_popen.return_value.wait.return_value = 0
d = tempfile.mkdtemp()
@@ -237,7 +237,7 @@ class AnsibleCommonTestCase(unittest.TestCase):
@mock.patch('{}.NamedTemporaryFile'.format(PREFIX))
@mock.patch('{}.Popen'.format(PREFIX))
@mock.patch('{}.open'.format(PREFIX))
- def test_execute_ansible_check(self, mock_open, mock_popen, mock_tmp):
+ def test_execute_ansible_check(self, _, mock_popen, __):
mock_popen.return_value.communicate.return_value = "", ""
mock_popen.return_value.wait.return_value = 0
d = tempfile.mkdtemp()
diff --git a/tests/unit/common/test_httpClient.py b/yardstick/tests/unit/common/test_httpClient.py
index eb09d1a52..eb09d1a52 100644
--- a/tests/unit/common/test_httpClient.py
+++ b/yardstick/tests/unit/common/test_httpClient.py
diff --git a/tests/unit/common/test_openstack_utils.py b/yardstick/tests/unit/common/test_openstack_utils.py
index b3dc2d9c4..bf468489e 100644
--- a/tests/unit/common/test_openstack_utils.py
+++ b/yardstick/tests/unit/common/test_openstack_utils.py
@@ -21,7 +21,7 @@ from yardstick.common import openstack_utils
class GetCredentialsTestCase(unittest.TestCase):
@mock.patch('yardstick.common.openstack_utils.os')
- def test_get_credentials(self, mock_os):
+ def test_get_credentials(self, _):
with mock.patch.dict('os.environ', {'OS_IDENTITY_API_VERSION': '2'},
clear=True):
openstack_utils.get_credentials()
diff --git a/tests/unit/common/test_process.py b/yardstick/tests/unit/common/test_process.py
index 1c6dfec27..1c6dfec27 100644
--- a/tests/unit/common/test_process.py
+++ b/yardstick/tests/unit/common/test_process.py
diff --git a/tests/unit/common/test_template_format.py b/yardstick/tests/unit/common/test_template_format.py
index 2a7d80b4d..44aa80333 100644
--- a/tests/unit/common/test_template_format.py
+++ b/yardstick/tests/unit/common/test_template_format.py
@@ -24,6 +24,9 @@ class TemplateFormatTestCase(unittest.TestCase):
def test_parse_to_value_exception(self):
+ # TODO(elfoley): Don't hide the error that occurs in
+ # template_format.parse
+ # TODO(elfoley): Separate these tests; one per error type
with mock.patch.object(yaml, 'load') as yaml_loader:
yaml_loader.side_effect = yaml.scanner.ScannerError()
self.assertRaises(ValueError, template_format.parse, 'FOOBAR')
diff --git a/tests/unit/common/test_utils.py b/yardstick/tests/unit/common/test_utils.py
index 452b93a56..452b93a56 100644
--- a/tests/unit/common/test_utils.py
+++ b/yardstick/tests/unit/common/test_utils.py
diff --git a/tests/unit/common/test_yaml_loader.py b/yardstick/tests/unit/common/test_yaml_loader.py
index 90cbb8157..90cbb8157 100644
--- a/tests/unit/common/test_yaml_loader.py
+++ b/yardstick/tests/unit/common/test_yaml_loader.py
diff --git a/tests/unit/test_ssh.py b/yardstick/tests/unit/test_ssh.py
index 88699fd85..dbaae8c37 100644
--- a/tests/unit/test_ssh.py
+++ b/yardstick/tests/unit/test_ssh.py
@@ -262,7 +262,7 @@ class SSHTestCase(unittest.TestCase):
self.test_client.execute.mock_calls)
@mock.patch("yardstick.ssh.paramiko")
- def test_send_command(self, mock_paramiko):
+ def test_send_command(self, _):
paramiko_sshclient = self.test_client._get_client()
with mock.patch.object(paramiko_sshclient, "exec_command") \
as mock_paramiko_exec_command: