diff options
| -rw-r--r-- | CONTRIBUTING.rst | 18 | ||||
| -rwxr-xr-x | docker/run_qtip.sh | 14 | ||||
| -rw-r--r-- | docs/designspec/compute_QPI.rst | 69 | ||||
| -rw-r--r-- | docs/designspec/index.rst | 1 | ||||
| -rw-r--r-- | qtip/__init__.py | 14 | ||||
| -rw-r--r-- | qtip/runner/__init__.py (renamed from __init__.py) | 0 | ||||
| -rw-r--r-- | qtip/runner/perftest.py | 27 | ||||
| -rw-r--r-- | qtip/runner/suite.py | 28 | ||||
| -rw-r--r-- | qtip/runner/test_plan.py | 28 | ||||
| -rw-r--r-- | scripts/qtip.py (renamed from qtip.py) | 0 | ||||
| -rw-r--r-- | setup.cfg | 22 | ||||
| -rw-r--r-- | setup.py | 16 | ||||
| -rw-r--r-- | tests/__init__.py | 0 | ||||
| -rw-r--r-- | tests/data/output/hosts (renamed from tests/output/hosts) | 0 | ||||
| -rw-r--r-- | tests/data/schema/test_bm_schema.yaml (renamed from tests/schema/test_bm_schema.yaml) | 0 | ||||
| -rw-r--r-- | tests/data/schema/test_vm_schema.yaml (renamed from tests/schema/test_vm_schema.yaml) | 0 | ||||
| -rw-r--r-- | tests/data/test_plan/bm_ping.yaml (renamed from tests/test_case/bm_ping.yaml) | 0 | ||||
| -rw-r--r-- | tests/data/test_plan/bm_with_proxy.yaml (renamed from tests/test_case/bm_with_proxy.yaml) | 0 | ||||
| -rw-r--r-- | tests/data/test_plan/bm_without_proxy.yaml (renamed from tests/test_case/bm_without_proxy.yaml) | 0 | ||||
| -rw-r--r-- | tests/data/test_plan/vm.yaml (renamed from tests/test_case/vm.yaml) | 0 | ||||
| -rw-r--r-- | tests/data/test_plan/vm_error.yaml (renamed from tests/test_case/vm_error.yaml) | 0 | ||||
| -rw-r--r-- | tests/functional/__init__.py | 0 | ||||
| -rw-r--r-- | tests/functional/yaml_schema_test.py (renamed from tests/yaml_schema_test.py) | 0 | ||||
| -rw-r--r-- | tests/unit/__init__.py | 0 | ||||
| -rw-r--r-- | tests/unit/api/__init__.py | 0 | ||||
| -rw-r--r-- | tests/unit/api/qtip_server_test.py (renamed from tests/qtip_server_test.py) | 0 | ||||
| -rw-r--r-- | tests/unit/cli/__init__.py | 0 | ||||
| -rw-r--r-- | tests/unit/runner/__init__.py | 0 | ||||
| -rw-r--r-- | tests/unit/runner/perftest_test.py | 13 | ||||
| -rw-r--r-- | tests/unit/runner/suite_test.py | 13 | ||||
| -rw-r--r-- | tests/unit/runner/test_plan_test.py | 13 | ||||
| -rw-r--r-- | tests/unit/utils/__init__.py | 0 | ||||
| -rw-r--r-- | tests/unit/utils/ansible_api_test.py (renamed from tests/ansible_api_test.py) | 0 | ||||
| -rw-r--r-- | tests/unit/utils/args_handler_test.py (renamed from tests/args_handler_test.py) | 0 | ||||
| -rw-r--r-- | tests/unit/utils/cli_test.py (renamed from tests/cli_test.py) | 0 | ||||
| -rw-r--r-- | tests/unit/utils/create_zones_test.py (renamed from tests/create_zones_test.py) | 0 | ||||
| -rw-r--r-- | tests/unit/utils/driver_test.py (renamed from tests/driver_test.py) | 0 | ||||
| -rw-r--r-- | tests/unit/utils/env_setup_test.py (renamed from tests/env_setup_test.py) | 0 | ||||
| -rw-r--r-- | tests/unit/utils/spawn_vm_test.py (renamed from tests/spawn_vm_test.py) | 0 | ||||
| -rw-r--r-- | tox.ini | 8 |
40 files changed, 261 insertions, 23 deletions
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6350a862..d10f1393 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -15,9 +15,26 @@ Peer review is the most important communication channel between developers. Every subtle change to the code or document **MUST** be reviewed before submission. +Add group ``qtip-reviewers`` in `gerrit`_ when you consider a patch set is ready. + Please make sure there is at least one ``+1`` or ``+2`` from others before submitting[#f1] a patch set. +Active Reviewers +================ + +Current list of active reviewers in gerrit group ``qtip-reviewers`` + +* Serena Feng <feng.xiaowei@zte.com.cn> +* Taseer Ahmed <taseer94@gmail.com> +* Yujun Zhang <zhang.yujunz@zte.com.cn> +* Zhifeng Jiang <jiang.zhifeng@zte.com.cn> +* Zhihui Wu <wu.zhihui1@zte.com.cn> + +By becoming an active reviewer, you agree to allow others to invite you as +reviewers in QTIP project freely. Any one in OPNFV community can apply to join +QTIP reviewers group or leave by submitting a patch on this document. + **************** Tasks and Issues **************** @@ -88,6 +105,7 @@ to submit. The current members are listed in `INFO`_. .. rubric:: Reference .. _Developer Getting Started: https://wiki.opnfv.org/display/DEV/Developer+Getting+Started +.. _gerrit: https://gerrit.opnfv.org/gerrit/#/q/project:+qtip .. _JIRA: https://jira.opnfv.org/browse/QTIP .. _OPNFV Releases: https://wiki.opnfv.org/display/SWREL .. _Issue Types: https://jira.opnfv.org/secure/ShowConstantsHelp.jspa?decorator=popup#IssueTypes diff --git a/docker/run_qtip.sh b/docker/run_qtip.sh index a7a20501..c2cf8c7b 100755 --- a/docker/run_qtip.sh +++ b/docker/run_qtip.sh @@ -1,20 +1,22 @@ #! /bin/bash +QTIP=scripts/qtip.py + run_test_suite() { if [ "$TEST_CASE" == "compute" ]; then - cd ${QTIP_DIR} && python qtip.py -l default -f compute + cd ${QTIP_DIR} && python ${QTIP} -l default -f compute cd ${QTIP_DIR} && python scripts/ref_results/suite_result.py compute elif [ "$TEST_CASE" == "storage" ]; then - cd ${QTIP_DIR} && python qtip.py -l default -f storage + cd ${QTIP_DIR} && python ${QTIP} -l default -f storage cd ${QTIP_DIR} && python scripts/ref_results/suite_result.py storage elif [ "$TEST_CASE" == "network" ]; then - cd ${QTIP_DIR} && python qtip.py -l default -f network + cd ${QTIP_DIR} && python ${QTIP} -l default -f network cd ${QTIP_DIR} && python scripts/ref_results/suite_result.py network elif [ "$TEST_CASE" == "all" ]; then - cd ${QTIP_DIR} && python qtip.py -l default -f compute - cd ${QTIP_DIR} && python qtip.py -l default -f storage - cd ${QTIP_DIR} && python qtip.py -l default -f network + cd ${QTIP_DIR} && python ${QTIP} -l default -f compute + cd ${QTIP_DIR} && python ${QTIP} -l default -f storage + cd ${QTIP_DIR} && python ${QTIP} -l default -f network cd ${QTIP_DIR} && python scripts/ref_results/suite_result.py compute cd ${QTIP_DIR} && python scripts/ref_results/suite_result.py storage diff --git a/docs/designspec/compute_QPI.rst b/docs/designspec/compute_QPI.rst new file mode 100644 index 00000000..2e5aa87c --- /dev/null +++ b/docs/designspec/compute_QPI.rst @@ -0,0 +1,69 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2016 ZTE Corp. + + +*********** +Compute QPI +*********** + +The compute QPI gives user an overall score for system compute performace. + +Summary +======= + +The compute QPI are calibrated a ZTE `E9000 <http://www.zte.com.cn/global/products/cocloud/cloud_computing/cloud_infrastructure/cloud_hw/429552>`_ server as a baseline with score of 2500 points. +Higher scores are better, with double the score indicating double the performance. +The compute QPI provides three different kinds of scores: + +* Workload Scores +* Section Scores +* Compute QPI Scores + +Baseline +======== + +ZTE E9000 server with an 2 Deca core Intel Xeon CPU processor,128560.0MB Memory. + +Workload Scores +=============== + +Each time a workload is executed QTIP calculates a score based on the computer's performance +compared to the baseline performance. + +Section Scores +============== + +QTIP uses a number of different tests, or workloads, to measure performance. +The workloads are divided into five different sections: + ++-----------------+--------------------------------------------------------------+------------------------------------------+ +| Section | Detail | Indication | ++=================+==============================================================+==========================================+ +| Integer | Integer workloads measure the integer instruction performace | All app relies on integer | +| | of host or vm by performing Dhrystone test. | performance | ++-----------------+--------------------------------------------------------------+------------------------------------------+ +| Floating point | Floating point workloads measure the floating pointperfo | Floating point performance is especially | +| | rmance by performing Whetstone test. | important in video games,digital content | +| | | creation applications. | ++-----------------+--------------------------------------------------------------+------------------------------------------+ +| Memory | Memory workloads measure memory bandwidth by performing | Software working with cipher large | +| | RamSpeed test. | amounts data relies on SSL Performace. | ++-----------------+--------------------------------------------------------------+------------------------------------------+ +| DPI | DPI workloads measure deep-packet inspection speed by | Software working with network packet | +| | performing nDPI test. | anlysis relies on DPI performance. | ++-----------------+--------------------------------------------------------------+------------------------------------------+ +| SSL | SSL Performance workloads measure cipher speeds by | Software working with cipher large | +| | using the OpenSSL tool. | amounts data relies on SSL Performace | ++-----------------+--------------------------------------------------------------+------------------------------------------+ + +A section score is the `geometric mean <https://en.wikipedia.org/wiki/Geometric_mean>`_ of all the workload scores for workloads +that are part of the section. These scores are useful for determining the performance of +the computer in a particular area. + +Compute QPI Scores +================== + +The compute QPI score is the `weighted arithmetic mean <https://en.wikipedia.org/wiki/Weighted_arithmetic_mean>`_ of the five section scores. +The compute QPI score provides a way to quickly compare performance across different +computers and different platforms without getting bogged down in details. diff --git a/docs/designspec/index.rst b/docs/designspec/index.rst index e9b3f9fd..d3bed217 100644 --- a/docs/designspec/index.rst +++ b/docs/designspec/index.rst @@ -11,3 +11,4 @@ QTIP Design Specifications :maxdepth: 2 dashboard.rst + compute_QPI.rst diff --git a/qtip/__init__.py b/qtip/__init__.py index e69de29b..228d036f 100644 --- a/qtip/__init__.py +++ b/qtip/__init__.py @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pbr.version + + +__version__ = pbr.version.VersionInfo( + 'qtip').version_string() diff --git a/__init__.py b/qtip/runner/__init__.py index e69de29b..e69de29b 100644 --- a/__init__.py +++ b/qtip/runner/__init__.py diff --git a/qtip/runner/perftest.py b/qtip/runner/perftest.py new file mode 100644 index 00000000..835f2a81 --- /dev/null +++ b/qtip/runner/perftest.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class PerfTest: + """A perf test collects raw performance metrics by running test tools""" + + def __init__(self): + pass + + @staticmethod + def list_all(): + """list all available perf tests""" + pass + + def desc(self): + """description of the perf test""" + pass + + def run(self): + pass diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py new file mode 100644 index 00000000..f0f2f63e --- /dev/null +++ b/qtip/runner/suite.py @@ -0,0 +1,28 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class Suite: + """A suite is consist of one or several perf tests and produces one QPI""" + + def __init__(self): + pass + + @staticmethod + def list_all(): + """list all available suites""" + pass + + def desc(self): + """description of the suite""" + pass + + def run(self): + """run included perftests in the suite""" + pass diff --git a/qtip/runner/test_plan.py b/qtip/runner/test_plan.py new file mode 100644 index 00000000..d20221df --- /dev/null +++ b/qtip/runner/test_plan.py @@ -0,0 +1,28 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class TestPlan: + """A test plan is consist of test configuration and selected test suites""" + + def __init__(self): + pass + + @staticmethod + def list_all(): + """list all available test plans""" + pass + + def desc(self): + """description of the test plan""" + pass + + def run(self): + """run included suites""" + pass diff --git a/qtip.py b/scripts/qtip.py index 66ff674f..66ff674f 100644 --- a/qtip.py +++ b/scripts/qtip.py diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..63286949 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[metadata] +name = qtip +summary = Platform Performance Benchmarking +description-file = + README.md +author = OPNFV +author-email = opnfv-tech-discuss@lists.opnfv.org +home-page = https://wiki.opnfv.org/display/qtip + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[entry_points] +console_scripts = + qtip = qtip.cli:entry + +[files] +packages = + qtip +data_files = + etc/qtip = benchmarks/* @@ -2,19 +2,7 @@ from setuptools import setup - setup( - name='qtip-cli', - version='0.1.dev0', - description='Platform Performance Benchmarking for OPNFV', - author='OPNFV', - author_email='zhang.yujunz@zte.com.cn', - install_requires=['click', 'pyyaml', 'prettytable'], - packages=['qtip.cli'], - entry_points={ - 'console_scripts': ['qtip=qtip.cli.entry:cli'] - }, - license='Apache-2.0', - keywords="performance benchmark opnfv", - url="https://wiki.opnfv.org/display/qtip" + setup_requires=['pbr>=1.9', 'setuptools>=17.1'], + pbr=True, ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/__init__.py diff --git a/tests/output/hosts b/tests/data/output/hosts index 9b47df0e..9b47df0e 100644 --- a/tests/output/hosts +++ b/tests/data/output/hosts diff --git a/tests/schema/test_bm_schema.yaml b/tests/data/schema/test_bm_schema.yaml index a7c27e3f..a7c27e3f 100644 --- a/tests/schema/test_bm_schema.yaml +++ b/tests/data/schema/test_bm_schema.yaml diff --git a/tests/schema/test_vm_schema.yaml b/tests/data/schema/test_vm_schema.yaml index 524f8fe4..524f8fe4 100644 --- a/tests/schema/test_vm_schema.yaml +++ b/tests/data/schema/test_vm_schema.yaml diff --git a/tests/test_case/bm_ping.yaml b/tests/data/test_plan/bm_ping.yaml index 41d696e2..41d696e2 100644 --- a/tests/test_case/bm_ping.yaml +++ b/tests/data/test_plan/bm_ping.yaml diff --git a/tests/test_case/bm_with_proxy.yaml b/tests/data/test_plan/bm_with_proxy.yaml index 1d73300b..1d73300b 100644 --- a/tests/test_case/bm_with_proxy.yaml +++ b/tests/data/test_plan/bm_with_proxy.yaml diff --git a/tests/test_case/bm_without_proxy.yaml b/tests/data/test_plan/bm_without_proxy.yaml index a9ae3b71..a9ae3b71 100644 --- a/tests/test_case/bm_without_proxy.yaml +++ b/tests/data/test_plan/bm_without_proxy.yaml diff --git a/tests/test_case/vm.yaml b/tests/data/test_plan/vm.yaml index 4c8453ca..4c8453ca 100644 --- a/tests/test_case/vm.yaml +++ b/tests/data/test_plan/vm.yaml diff --git a/tests/test_case/vm_error.yaml b/tests/data/test_plan/vm_error.yaml index f13d3a00..f13d3a00 100644 --- a/tests/test_case/vm_error.yaml +++ b/tests/data/test_plan/vm_error.yaml diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/functional/__init__.py diff --git a/tests/yaml_schema_test.py b/tests/functional/yaml_schema_test.py index a975dca6..a975dca6 100644 --- a/tests/yaml_schema_test.py +++ b/tests/functional/yaml_schema_test.py diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unit/__init__.py diff --git a/tests/unit/api/__init__.py b/tests/unit/api/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unit/api/__init__.py diff --git a/tests/qtip_server_test.py b/tests/unit/api/qtip_server_test.py index 96544c95..96544c95 100644 --- a/tests/qtip_server_test.py +++ b/tests/unit/api/qtip_server_test.py diff --git a/tests/unit/cli/__init__.py b/tests/unit/cli/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unit/cli/__init__.py diff --git a/tests/unit/runner/__init__.py b/tests/unit/runner/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unit/runner/__init__.py diff --git a/tests/unit/runner/perftest_test.py b/tests/unit/runner/perftest_test.py new file mode 100644 index 00000000..798afadd --- /dev/null +++ b/tests/unit/runner/perftest_test.py @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class TestPerfTest: + def test_list(self): + assert True diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py new file mode 100644 index 00000000..a2023cf8 --- /dev/null +++ b/tests/unit/runner/suite_test.py @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class TestSuite: + def test_list(self): + assert True diff --git a/tests/unit/runner/test_plan_test.py b/tests/unit/runner/test_plan_test.py new file mode 100644 index 00000000..81f618c7 --- /dev/null +++ b/tests/unit/runner/test_plan_test.py @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class TestTestPlan: + def test_list(self): + assert True diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unit/utils/__init__.py diff --git a/tests/ansible_api_test.py b/tests/unit/utils/ansible_api_test.py index 6b1afb44..6b1afb44 100644 --- a/tests/ansible_api_test.py +++ b/tests/unit/utils/ansible_api_test.py diff --git a/tests/args_handler_test.py b/tests/unit/utils/args_handler_test.py index b67fd993..b67fd993 100644 --- a/tests/args_handler_test.py +++ b/tests/unit/utils/args_handler_test.py diff --git a/tests/cli_test.py b/tests/unit/utils/cli_test.py index 86dd6b98..86dd6b98 100644 --- a/tests/cli_test.py +++ b/tests/unit/utils/cli_test.py diff --git a/tests/create_zones_test.py b/tests/unit/utils/create_zones_test.py index 8b1e97cc..8b1e97cc 100644 --- a/tests/create_zones_test.py +++ b/tests/unit/utils/create_zones_test.py diff --git a/tests/driver_test.py b/tests/unit/utils/driver_test.py index 432ce1ae..432ce1ae 100644 --- a/tests/driver_test.py +++ b/tests/unit/utils/driver_test.py diff --git a/tests/env_setup_test.py b/tests/unit/utils/env_setup_test.py index e28c6a18..e28c6a18 100644 --- a/tests/env_setup_test.py +++ b/tests/unit/utils/env_setup_test.py diff --git a/tests/spawn_vm_test.py b/tests/unit/utils/spawn_vm_test.py index ba237378..ba237378 100644 --- a/tests/spawn_vm_test.py +++ b/tests/unit/utils/spawn_vm_test.py @@ -10,17 +10,19 @@ skipsdist = True [testenv] usedevelop = True install_command = pip install -U {opts} {packages} -deps = +deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands= py.test \ --basetemp={envtmpdir} \ - {posargs} tests + {posargs} +setenv= + PYTHONPATH = {toxinidir} [testenv:pep8] deps = flake8 -commands = flake8 {toxinidir} +commands = flake8 {toxinidir} [flake8] # H803 skipped on purpose per list discussion. |