diff options
| -rw-r--r-- | docs/testing/user/userguide/opnfv_yardstick_tc084.rst | 140 | ||||
| -rw-r--r-- | tests/unit/common/__init__.py | 0 | ||||
| -rw-r--r-- | tools/coding-checks.sh | 9 | ||||
| -rwxr-xr-x | tools/run_tests.sh | 8 | ||||
| -rw-r--r-- | tox.ini | 2 | ||||
| -rwxr-xr-x | yardstick/tests/functional/test_cli_runner.py (renamed from tests/functional/test_cli_runner.py) | 4 | ||||
| -rwxr-xr-x | yardstick/tests/functional/test_cli_scenario.py (renamed from tests/functional/test_cli_scenario.py) | 4 | ||||
| -rwxr-xr-x | yardstick/tests/functional/utils.py (renamed from tests/functional/utils.py) | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/apiserver/__init__.py (renamed from tests/unit/apiserver/__init__.py) | 17 | ||||
| -rw-r--r--[-rwxr-xr-x] | yardstick/tests/unit/apiserver/resources/__init__.py (renamed from tests/functional/__init__.py) | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/apiserver/resources/test_env_action.py (renamed from tests/unit/apiserver/resources/test_env_action.py) | 3 | ||||
| -rw-r--r-- | yardstick/tests/unit/apiserver/utils/test_influx.py (renamed from tests/unit/apiserver/utils/test_influx.py) | 8 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/__init__.py (renamed from tests/unit/apiserver/resources/__init__.py) | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/config_sample.yaml (renamed from tests/unit/common/config_sample.yaml) | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/test_ansible_common.py (renamed from tests/unit/common/test_ansible_common.py) | 12 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/test_httpClient.py (renamed from tests/unit/common/test_httpClient.py) | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/test_openstack_utils.py (renamed from tests/unit/common/test_openstack_utils.py) | 2 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/test_process.py (renamed from tests/unit/common/test_process.py) | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/test_template_format.py (renamed from tests/unit/common/test_template_format.py) | 3 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/test_utils.py (renamed from tests/unit/common/test_utils.py) | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/common/test_yaml_loader.py (renamed from tests/unit/common/test_yaml_loader.py) | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/test_ssh.py (renamed from tests/unit/test_ssh.py) | 2 |
22 files changed, 186 insertions, 28 deletions
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc084.rst b/docs/testing/user/userguide/opnfv_yardstick_tc084.rst new file mode 100644 index 000000000..2e7b28e25 --- /dev/null +++ b/docs/testing/user/userguide/opnfv_yardstick_tc084.rst @@ -0,0 +1,140 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Huawei Technologies Co.,Ltd and others. + +************************************* +Yardstick Test Case Description TC084 +************************************* + +.. _spec_cpu_2006: https://www.spec.org/cpu2006/ + ++-----------------------------------------------------------------------------+ +|Compute Performance | +| | ++--------------+--------------------------------------------------------------+ +|test case id | OPNFV_YARDSTICK_TC084_SPEC CPU 2006 FOR VM | +| | | ++--------------+--------------------------------------------------------------+ +|metric | compute-intensive performance | +| | | ++--------------+--------------------------------------------------------------+ +|test purpose | The purpose of TC084 is to evaluate the IaaS compute | +| | performance by using SPEC CPU 2006 benchmark. The SPEC CPU | +| | 2006 benchmark has several different ways to measure | +| | computer performance. One way is to measure how fast the | +| | computer completes a single task; this is called a speed | +| | measurement. Another way is to measure how many tasks | +| | computer can accomplish in a certain amount of time; this is | +| | called a throughput, capacity or rate measurement. | +| | | ++--------------+--------------------------------------------------------------+ +|test tool | SPEC CPU 2006 | +| | | +| | The SPEC CPU 2006 benchmark is SPEC's industry-standardized, | +| | CPU-intensive benchmark suite, stressing a system's | +| | processor, memory subsystem and compiler. This benchmark | +| | suite includes the SPECint benchmarks and the SPECfp | +| | benchmarks. The SPECint 2006 benchmark contains 12 different | +| | benchmark tests and the SPECfp 2006 benchmark contains 19 | +| | different benchmark tests. | +| | | +| | SPEC CPU 2006 is not always part of a Linux distribution. | +| | SPEC requires that users purchase a license and agree with | +| | their terms and conditions. For this test case, users must | +| | manually download cpu2006-1.2.iso from the SPEC website and | +| | save it under the yardstick/resources folder (e.g. /home/ | +| | opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso) | +| | SPEC CPU® 2006 benchmark is available for purchase via the | +| | SPEC order form (https://www.spec.org/order.html). | +| | | ++--------------+--------------------------------------------------------------+ +|test | This test case uses SPEC CPU 2006 benchmark to measure | +|description | compute-intensive performance of VMs. | +| | | ++--------------+--------------------------------------------------------------+ +|configuration | file: opnfv_yardstick_tc084.yaml | +| | | +| | benchmark_subset is set to int. | +| | | +| | SLA is not available in this test case. | +| | | ++--------------+--------------------------------------------------------------+ +|applicability | Test can be configured with different: | +| | | +| | * benchmark_subset - a subset of SPEC CPU 2006 benchmarks | +| | to run; | +| | * SPECint_benchmark - a SPECint benchmark to run; | +| | * SPECint_benchmark - a SPECfp benchmark to run; | +| | * output_format - desired report format; | +| | * runspec_config - SPEC CPU 2006 config file provided to | +| | the runspec binary; | +| | * runspec_iterations - the number of benchmark iterations | +| | to execute. For a reportable run, must be 3; | +| | * runspec_tune - tuning to use (base, peak, or all). For a | +| | reportable run, must be either base or all. Reportable | +| | runs do base first, then (optionally) peak; | +| | * runspec_size - size of input data to run (test, train, or | +| | ref). Reportable runs ensure that your binaries can | +| | produce correct results with the test and train workloads | +| | | ++--------------+--------------------------------------------------------------+ +|usability | This test case is used for executing SPEC CPU 2006 benchmark | +| | on virtual machines. The SPECint 2006 benchmark takes | +| | approximately 5 hours. (The time may vary due to different | +| | VM cpu configurations) | +| | | ++--------------+--------------------------------------------------------------+ +|references | spec_cpu_2006_ | +| | | +| | ETSI-NFV-TST001 | +| | | ++--------------+--------------------------------------------------------------+ +|pre-test | To run and install SPEC CPU 2006, the following are | +|conditions | required: | +| | * For SPECint 2006: Both C99 and C++98 compilers are | +| | installed in VM images; | +| | * For SPECfp 2006: All three of C99, C++98 and Fortran-95 | +| | compilers installed in VM images; | +| | * At least 4GB of disk space availabile on VM. | +| | | +| | gcc 4.8.* and g++ 4.8.* version have been tested in Ubuntu | +| | 14.04, Ubuntu 16.04 and Redhat Enterprise Linux 7.4 image. | +| | Higher gcc and g++ version may cause compiling error. | +| | | +| | For more SPEC CPU 2006 dependencies please visit | +| | (https://www.spec.org/cpu2006/Docs/techsupport.html) | +| | | ++--------------+--------------------------------------------------------------+ +|test sequence | description and expected result | +| | | ++--------------+--------------------------------------------------------------+ +|step 1 | cpu2006-1.2.iso has been saved under the yardstick/resources | +| | folder (e.g. /home/opnfv/repos/yardstick/yardstick/resources | +| | /cpu2006-1.2.iso). Additionally, to use your custom runspec | +| | config file you can save it under the yardstick/resources/ | +| | files folder and specify the config file name in the | +| | runspec_config parameter. | +| | | ++--------------+--------------------------------------------------------------+ +|step 2 | Upload SPEC CPU 2006 ISO to the target VM using scp and | +| | install SPEC CPU 2006. | +| | | ++--------------+--------------------------------------------------------------+ +|step 3 | Connect to the target server using SSH. | +| | If custom runspec config file is used, copy this file from | +| | yardstick to the target VM via the SSH tunnel. | +| | | ++--------------+--------------------------------------------------------------+ +|step 4 | SPEC CPU 2006 benchmark is invoked and SPEC CPU 2006 metrics | +| | are generated. | +| | | ++--------------+--------------------------------------------------------------+ +|step 5 | Text, HTML, CSV, PDF, and Configuration file outputs for the | +| | SPEC CPU 2006 metrics are fetched from the VM and stored | +| | under /tmp/result folder. | +| | | ++--------------+--------------------------------------------------------------+ +|test verdict | None. SPEC CPU 2006 results are collected and stored. | +| | | ++--------------+--------------------------------------------------------------+ diff --git a/tests/unit/common/__init__.py b/tests/unit/common/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/tests/unit/common/__init__.py +++ /dev/null diff --git a/tools/coding-checks.sh b/tools/coding-checks.sh index 4ee909988..615f2c3de 100644 --- a/tools/coding-checks.sh +++ b/tools/coding-checks.sh @@ -28,6 +28,7 @@ process_options () { run_pylint () { local target="${scriptargs:-all}" + local output_format="" if [ "$target" = "all" ]; then files="ansible api tests yardstick" @@ -37,11 +38,15 @@ run_pylint () { *) echo "$target is an unrecognized basecommit"; exit 1;; esac fi - + # make Jenkins output parseable because Jenkins doesn't handle color + # enventually we should use the Jenkins Pylint plugin or other tools + if [ -n "${BRANCH:-}" ] ; then + output_format="--output-format=parseable" + fi echo "Running pylint..." echo "You can speed this up by running it on 'HEAD~[0-9]' (e.g. HEAD~0, this change only)..." if [ -n "${files}" ]; then - pylint --rcfile=.pylintrc ${files} + pylint --rcfile=.pylintrc ${output_format} ${files} else echo "No python changes in this commit, pylint check not required." exit 0 diff --git a/tools/run_tests.sh b/tools/run_tests.sh index ddb5e55af..32c4f19e4 100755 --- a/tools/run_tests.sh +++ b/tools/run_tests.sh @@ -29,9 +29,16 @@ run_tests() { echo "Running unittest ... " if [ $FILE_OPTION == "f" ]; then python -m unittest discover -v -s tests/unit > $logfile 2>&1 + if [ $? -ne 0 ]; then + echo "FAILED, results in $logfile" + exit 1 + fi python -m unittest discover -v -s yardstick/tests/unit >> $logfile 2>&1 else python -m unittest discover -v -s tests/unit + if [ $? -ne 0 ]; then + exit 1 + fi python -m unittest discover -v -s yardstick/tests/unit fi @@ -56,7 +63,6 @@ run_functional_test() { mkdir -p .testrepository python -m subunit.run discover yardstick/tests/functional > .testrepository/subunit.log - python -m subunit.run discover tests/functional >> .testrepository/subunit.log subunit2pyunit < .testrepository/subunit.log EXIT_CODE=$? @@ -5,7 +5,7 @@ envlist = py{27,3},pep8,functional{,-py3},coverage [testenv] usedevelop=True -passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY +passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY BRANCH deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt diff --git a/tests/functional/test_cli_runner.py b/yardstick/tests/functional/test_cli_runner.py index 620edc396..2f2d7fef2 100755 --- a/tests/functional/test_cli_runner.py +++ b/yardstick/tests/functional/test_cli_runner.py @@ -7,11 +7,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - -from __future__ import absolute_import import unittest -from tests.functional import utils +from yardstick.tests.functional import utils class RunnerTestCase(unittest.TestCase): diff --git a/tests/functional/test_cli_scenario.py b/yardstick/tests/functional/test_cli_scenario.py index 63b533b85..7aaacad0a 100755 --- a/tests/functional/test_cli_scenario.py +++ b/yardstick/tests/functional/test_cli_scenario.py @@ -7,11 +7,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - -from __future__ import absolute_import import unittest -from tests.functional import utils +from yardstick.tests.functional import utils class ScenarioTestCase(unittest.TestCase): diff --git a/tests/functional/utils.py b/yardstick/tests/functional/utils.py index d889c0dfa..d889c0dfa 100755 --- a/tests/functional/utils.py +++ b/yardstick/tests/functional/utils.py diff --git a/tests/unit/apiserver/__init__.py b/yardstick/tests/unit/apiserver/__init__.py index 5e1ed2ea1..44d163429 100644 --- a/tests/unit/apiserver/__init__.py +++ b/yardstick/tests/unit/apiserver/__init__.py @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2017 +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +"""Tests for yardstick/api/server.py""" from __future__ import absolute_import import mock @@ -12,14 +21,16 @@ from yardstick.common import constants as consts class APITestCase(unittest.TestCase): - + """Tests for the YardStick API server""" def setUp(self): self.db_fd, self.db_path = tempfile.mkstemp() consts.SQLITE = 'sqlite:///{}'.format(self.db_path) - # server calls gethostbyname which takes 4 seconds, and we should mock it anyway + # server calls gethostbyname which takes 4 seconds, and we should mock + # it anyway self.socket_mock = mock.patch.dict("sys.modules", {"socket": mock.MagicMock( - **{"gethostbyname.return_value": "127.0.0.1", "gethostname.return_value": "localhost"})}) + **{"gethostbyname.return_value": "127.0.0.1", + "gethostname.return_value": "localhost"})}) self.socket_mock.start() try: from api import server diff --git a/tests/functional/__init__.py b/yardstick/tests/unit/apiserver/resources/__init__.py index e69de29bb..e69de29bb 100755..100644 --- a/tests/functional/__init__.py +++ b/yardstick/tests/unit/apiserver/resources/__init__.py diff --git a/tests/unit/apiserver/resources/test_env_action.py b/yardstick/tests/unit/apiserver/resources/test_env_action.py index 5417ad953..b7bfe294d 100644 --- a/tests/unit/apiserver/resources/test_env_action.py +++ b/yardstick/tests/unit/apiserver/resources/test_env_action.py @@ -6,12 +6,11 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from __future__ import absolute_import import time import unittest -from tests.unit.apiserver import APITestCase +from yardstick.tests.unit.apiserver import APITestCase class EnvTestCase(APITestCase): diff --git a/tests/unit/apiserver/utils/test_influx.py b/yardstick/tests/unit/apiserver/utils/test_influx.py index aff0cab5c..883608bb2 100644 --- a/tests/unit/apiserver/utils/test_influx.py +++ b/yardstick/tests/unit/apiserver/utils/test_influx.py @@ -6,13 +6,11 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from __future__ import absolute_import import unittest import mock from api.utils import influx - -import six.moves.configparser as ConfigParser +from six.moves import configparser as ConfigParser class GetDataDbClientTestCase(unittest.TestCase): @@ -25,7 +23,7 @@ class GetDataDbClientTestCase(unittest.TestCase): mock_parser.NoOptionError = ConfigParser.NoOptionError try: influx.get_data_db_client() - except Exception as e: + except Exception as e: # pylint: disable=broad-except self.assertIsInstance(e, RuntimeError) @@ -50,7 +48,7 @@ class QueryTestCase(unittest.TestCase): try: sql = 'select * form tasklist' influx.query(sql) - except Exception as e: + except Exception as e: # pylint: disable=broad-except self.assertIsInstance(e, RuntimeError) diff --git a/tests/unit/apiserver/resources/__init__.py b/yardstick/tests/unit/common/__init__.py index e69de29bb..e69de29bb 100644 --- a/tests/unit/apiserver/resources/__init__.py +++ b/yardstick/tests/unit/common/__init__.py diff --git a/tests/unit/common/config_sample.yaml b/yardstick/tests/unit/common/config_sample.yaml index 09218cc79..09218cc79 100644 --- a/tests/unit/common/config_sample.yaml +++ b/yardstick/tests/unit/common/config_sample.yaml diff --git a/tests/unit/common/test_ansible_common.py b/yardstick/tests/unit/common/test_ansible_common.py index 1ef8eee5f..89ea128af 100644 --- a/tests/unit/common/test_ansible_common.py +++ b/yardstick/tests/unit/common/test_ansible_common.py @@ -49,7 +49,7 @@ class OverwriteDictTestCase(unittest.TestCase): class FilenameGeneratorTestCase(unittest.TestCase): @mock.patch('{}.NamedTemporaryFile'.format(PREFIX)) - def test__handle_existing_file(self, mock_tmp): + def test__handle_existing_file(self, _): ansible_common.FileNameGenerator._handle_existing_file("/dev/null") def test_get_generator_from_file(self): @@ -184,7 +184,7 @@ class AnsibleCommonTestCase(unittest.TestCase): self.assertEqual(a.deploy_dir, "d") @mock.patch('{}.open'.format(PREFIX)) - def test__gen_ansible_playbook_file_list(self, mock_open): + def test__gen_ansible_playbook_file_list(self, _): d = tempfile.mkdtemp() try: a = ansible_common.AnsibleCommon({}) @@ -194,7 +194,7 @@ class AnsibleCommonTestCase(unittest.TestCase): @mock.patch('{}.NamedTemporaryFile'.format(PREFIX)) @mock.patch('{}.open'.format(PREFIX)) - def test__gen_ansible_inventory_file(self, mock_open, mock_tmp): + def test__gen_ansible_inventory_file(self, _, __): nodes = [{ "name": "name", "user": "user", "password": "PASS", "role": "role", @@ -213,7 +213,7 @@ class AnsibleCommonTestCase(unittest.TestCase): @mock.patch('{}.NamedTemporaryFile'.format(PREFIX)) @mock.patch('{}.open'.format(PREFIX)) - def test__gen_ansible_playbook_file_list_multiple(self, mock_open, mock_tmp): + def test__gen_ansible_playbook_file_list_multiple(self, _, __): d = tempfile.mkdtemp() try: a = ansible_common.AnsibleCommon({}) @@ -224,7 +224,7 @@ class AnsibleCommonTestCase(unittest.TestCase): @mock.patch('{}.NamedTemporaryFile'.format(PREFIX)) @mock.patch('{}.Popen'.format(PREFIX)) @mock.patch('{}.open'.format(PREFIX)) - def test_do_install_tmp_dir(self, mock_open, mock_popen, mock_tmp): + def test_do_install_tmp_dir(self, _, mock_popen, __): mock_popen.return_value.communicate.return_value = "", "" mock_popen.return_value.wait.return_value = 0 d = tempfile.mkdtemp() @@ -237,7 +237,7 @@ class AnsibleCommonTestCase(unittest.TestCase): @mock.patch('{}.NamedTemporaryFile'.format(PREFIX)) @mock.patch('{}.Popen'.format(PREFIX)) @mock.patch('{}.open'.format(PREFIX)) - def test_execute_ansible_check(self, mock_open, mock_popen, mock_tmp): + def test_execute_ansible_check(self, _, mock_popen, __): mock_popen.return_value.communicate.return_value = "", "" mock_popen.return_value.wait.return_value = 0 d = tempfile.mkdtemp() diff --git a/tests/unit/common/test_httpClient.py b/yardstick/tests/unit/common/test_httpClient.py index eb09d1a52..eb09d1a52 100644 --- a/tests/unit/common/test_httpClient.py +++ b/yardstick/tests/unit/common/test_httpClient.py diff --git a/tests/unit/common/test_openstack_utils.py b/yardstick/tests/unit/common/test_openstack_utils.py index b3dc2d9c4..bf468489e 100644 --- a/tests/unit/common/test_openstack_utils.py +++ b/yardstick/tests/unit/common/test_openstack_utils.py @@ -21,7 +21,7 @@ from yardstick.common import openstack_utils class GetCredentialsTestCase(unittest.TestCase): @mock.patch('yardstick.common.openstack_utils.os') - def test_get_credentials(self, mock_os): + def test_get_credentials(self, _): with mock.patch.dict('os.environ', {'OS_IDENTITY_API_VERSION': '2'}, clear=True): openstack_utils.get_credentials() diff --git a/tests/unit/common/test_process.py b/yardstick/tests/unit/common/test_process.py index 1c6dfec27..1c6dfec27 100644 --- a/tests/unit/common/test_process.py +++ b/yardstick/tests/unit/common/test_process.py diff --git a/tests/unit/common/test_template_format.py b/yardstick/tests/unit/common/test_template_format.py index 2a7d80b4d..44aa80333 100644 --- a/tests/unit/common/test_template_format.py +++ b/yardstick/tests/unit/common/test_template_format.py @@ -24,6 +24,9 @@ class TemplateFormatTestCase(unittest.TestCase): def test_parse_to_value_exception(self): + # TODO(elfoley): Don't hide the error that occurs in + # template_format.parse + # TODO(elfoley): Separate these tests; one per error type with mock.patch.object(yaml, 'load') as yaml_loader: yaml_loader.side_effect = yaml.scanner.ScannerError() self.assertRaises(ValueError, template_format.parse, 'FOOBAR') diff --git a/tests/unit/common/test_utils.py b/yardstick/tests/unit/common/test_utils.py index 452b93a56..452b93a56 100644 --- a/tests/unit/common/test_utils.py +++ b/yardstick/tests/unit/common/test_utils.py diff --git a/tests/unit/common/test_yaml_loader.py b/yardstick/tests/unit/common/test_yaml_loader.py index 90cbb8157..90cbb8157 100644 --- a/tests/unit/common/test_yaml_loader.py +++ b/yardstick/tests/unit/common/test_yaml_loader.py diff --git a/tests/unit/test_ssh.py b/yardstick/tests/unit/test_ssh.py index 88699fd85..dbaae8c37 100644 --- a/tests/unit/test_ssh.py +++ b/yardstick/tests/unit/test_ssh.py @@ -262,7 +262,7 @@ class SSHTestCase(unittest.TestCase): self.test_client.execute.mock_calls) @mock.patch("yardstick.ssh.paramiko") - def test_send_command(self, mock_paramiko): + def test_send_command(self, _): paramiko_sshclient = self.test_client._get_client() with mock.patch.object(paramiko_sshclient, "exec_command") \ as mock_paramiko_exec_command: |