diff options
| -rw-r--r-- | docker/Dockerfile | 2 | ||||
| -rw-r--r-- | docs/proposal/dashboard.rst (renamed from docs/testing/developer/design/dashboard.rst) | 0 | ||||
| -rw-r--r-- | docs/proposal/integration_with_yardstick.rst (renamed from docs/testing/developer/design/integration_with_yardstick.rst) | 0 | ||||
| -rw-r--r-- | docs/release/release-notes/danube.rst | 171 | ||||
| -rw-r--r-- | docs/testing/.gitkeep | 0 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/_assets/sequence.png | bin | 0 -> 44132 bytes | |||
| -rw-r--r-- | docs/testing/developer/devguide/_assets/standalone.png | bin | 0 -> 47515 bytes | |||
| -rw-r--r-- | docs/testing/developer/devguide/api.rst | 2 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/arch.rst | 24 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/cli.rst (renamed from docs/testing/developer/design/cli.rst) | 0 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/compute-qpi.rst (renamed from docs/testing/developer/design/compute_QPI.rst) | 0 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/index.rst (renamed from docs/testing/developer/design/index.rst) | 7 | ||||
| -rw-r--r-- | docs/testing/user/.gitkeep | 0 | ||||
| -rw-r--r-- | docs/testing/user/installation/.gitkeep | 0 | ||||
| -rw-r--r-- | docs/testing/user/installation/README | 1 | ||||
| -rw-r--r-- | docs/testing/user/scenarios/.gitkeep | 0 | ||||
| -rw-r--r-- | docs/testing/user/scenarios/README | 1 | ||||
| -rw-r--r-- | docs/testing/user/userguide/index.rst | 5 | ||||
| -rw-r--r-- | docs/testing/user/userguide/qpi-compute.rst | 104 | ||||
| -rw-r--r-- | legacy/docs/_01-compute.rst (renamed from docs/testing/user/userguide/_01-compute.rst) | 0 | ||||
| -rw-r--r-- | legacy/docs/_02-network.rst (renamed from docs/testing/user/userguide/_02-network.rst) | 0 | ||||
| -rw-r--r-- | legacy/docs/_03-storage.rst (renamed from docs/testing/user/userguide/_03-storage.rst) | 0 | ||||
| -rw-r--r-- | legacy/docs/_testcase_description.rst (renamed from docs/testing/user/userguide/_testcase_description.rst) | 0 | ||||
| -rw-r--r-- | legacy/docs/annex.rst (renamed from docs/testing/user/userguide/annex.rst) | 0 | ||||
| -rw-r--r-- | legacy/docs/apidocs/qtip_restful_api.rst (renamed from docs/testing/developer/design/apidocs/qtip_restful_api.rst) | 0 | ||||
| -rw-r--r-- | legacy/docs/benchmark-suites.rst (renamed from docs/testing/user/userguide/benchmark-suites.rst) | 0 | ||||
| -rw-r--r-- | legacy/docs/index.rst (renamed from docs/testing/developer/design/apidocs/index.rst) | 0 | ||||
| -rw-r--r-- | legacy/docs/introduction.rst (renamed from docs/testing/user/userguide/introduction.rst) | 0 |
28 files changed, 310 insertions, 7 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index d609273d..d6f8fdd6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -62,7 +62,7 @@ RUN git config --global http.sslVerify false RUN git clone -b $BRANCH https://gerrit.opnfv.org/gerrit/qtip $REPOS_DIR/qtip RUN git clone https://gerrit.opnfv.org/gerrit/releng $REPOS_DIR/releng -RUN pip install -U -r $REPOS_DIR/qtip/requirements.txt +RUN cd $REPOS_DIR/qtip && pip install -U . #Config supervisor RUN mkdir -p /var/log/supervisor diff --git a/docs/testing/developer/design/dashboard.rst b/docs/proposal/dashboard.rst index 60c4720d..60c4720d 100644 --- a/docs/testing/developer/design/dashboard.rst +++ b/docs/proposal/dashboard.rst diff --git a/docs/testing/developer/design/integration_with_yardstick.rst b/docs/proposal/integration_with_yardstick.rst index a8298d6f..a8298d6f 100644 --- a/docs/testing/developer/design/integration_with_yardstick.rst +++ b/docs/proposal/integration_with_yardstick.rst diff --git a/docs/release/release-notes/danube.rst b/docs/release/release-notes/danube.rst new file mode 100644 index 00000000..b50fb909 --- /dev/null +++ b/docs/release/release-notes/danube.rst @@ -0,0 +1,171 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 + + +This document provides the release notes for <RELEASE> of <COMPONENT>. + +.. contents:: + :depth: 3 + :local: + + +Version history +--------------- + ++--------------------+--------------------+--------------------+--------------------+ +| **Date** | **Ver.** | **Author** | **Comment** | +| | | | | ++--------------------+--------------------+--------------------+--------------------+ +| 2017-03-14 | TODO(yujunz): tag | Yujun Zhang | First draft | +| | | | | ++--------------------+--------------------+--------------------+--------------------+ + +Important notes +=============== + +QTIP is totally reworked in Danube release. The legacy benchmarks released in Brahmaputra (compute, network and storage) +are deprecated. + +Summary +======= + +QTIP Danube release introduces **QPI**, a.k.a. QTIP Performance Index, which is calculated from metrics collected in +performance tests. + +A PoC of compute qpi benchmark plan is provided as a sample use case. + +Available benchmark plans can be listed, shown and executed from `qtip-cli`, the command line interpreter of QTIP. + +Release Data +============ + ++--------------------------------------+--------------------------------------+ +| **Project** | QTIP | +| | | ++--------------------------------------+--------------------------------------+ +| **Repo/commit-ID** | qtip/TBD | +| | | ++--------------------------------------+--------------------------------------+ +| **Release designation** | Danube 1.0 | +| | | ++--------------------------------------+--------------------------------------+ +| **Release date** | TODO(yujunz): fill date | +| | | ++--------------------------------------+--------------------------------------+ +| **Purpose of the delivery** | OPNFV quality assurance | +| | | ++--------------------------------------+--------------------------------------+ + +Version change +^^^^^^^^^^^^^^ + +Module version changes +~~~~~~~~~~~~~~~~~~~~~~ + +N/A + +Document version changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +N/A + +Reason for version +^^^^^^^^^^^^^^^^^^ +Feature additions +~~~~~~~~~~~~~~~~~ + +**JIRA BACK-LOG:** + +** TODO(yujunz) pull data from JIRA ** + ++--------------------------------------+--------------------------------------+ +| **JIRA REFERENCE** | **SLOGAN** | +| | | ++--------------------------------------+--------------------------------------+ +| | | ++--------------------------------------+--------------------------------------+ +| | | ++--------------------------------------+--------------------------------------+ + +Bug corrections +~~~~~~~~~~~~~~~ + +**JIRA TICKETS:** + +** TODO(yujunz) pull data from JIRA ** + ++--------------------------------------+--------------------------------------+ +| **JIRA REFERENCE** | **SLOGAN** | +| | | ++--------------------------------------+--------------------------------------+ +| | | ++--------------------------------------+--------------------------------------+ +| | | ++--------------------------------------+--------------------------------------+ + +Deliverables +------------ + +Software deliverables +^^^^^^^^^^^^^^^^^^^^^ + +** TODO(yujunz) docker image, pip package ** + +Documentation deliverables +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +** TODO(yujunz) links to qtip document publish ** + +Known Limitations, Issues and Workarounds +========================================= + +System Limitations +^^^^^^^^^^^^^^^^^^ + +None + +Known issues +^^^^^^^^^^^^ + +** TODO(yujunz) pull data from JIRA ** + +**JIRA TICKETS:** + ++--------------------------------------+--------------------------------------+ +| **JIRA REFERENCE** | **SLOGAN** | +| | | ++--------------------------------------+--------------------------------------+ +| | | ++--------------------------------------+--------------------------------------+ +| | | ++--------------------------------------+--------------------------------------+ + +Workarounds +^^^^^^^^^^^ + +N/A + +Test Result +=========== + +QTIP has undergone QA test runs with the following results: + ++--------------------------------------+--------------------------------------+ +| **TEST-SUITES** | **Results:** | +| | | ++--------------------------------------+--------------------------------------+ +| qtip-verify-danube | 72/72 passed | +| | | +| | 73% lines of code covered | ++--------------------------------------+--------------------------------------+ +| qtip-daily-fuel-zte-pod3-danube | Last 7 build fails | +| | | +| | blocked by the failure of zte-pod3 | ++--------------------------------------+--------------------------------------+ + +References +========== + +For more information on the OPNFV Danube release, please see: + +http://opnfv.org/danube diff --git a/docs/testing/.gitkeep b/docs/testing/.gitkeep deleted file mode 100644 index e69de29b..00000000 --- a/docs/testing/.gitkeep +++ /dev/null diff --git a/docs/testing/developer/devguide/_assets/sequence.png b/docs/testing/developer/devguide/_assets/sequence.png Binary files differnew file mode 100644 index 00000000..b15458bf --- /dev/null +++ b/docs/testing/developer/devguide/_assets/sequence.png diff --git a/docs/testing/developer/devguide/_assets/standalone.png b/docs/testing/developer/devguide/_assets/standalone.png Binary files differnew file mode 100644 index 00000000..50222b5b --- /dev/null +++ b/docs/testing/developer/devguide/_assets/standalone.png diff --git a/docs/testing/developer/devguide/api.rst b/docs/testing/developer/devguide/api.rst new file mode 100644 index 00000000..eb2b0d67 --- /dev/null +++ b/docs/testing/developer/devguide/api.rst @@ -0,0 +1,2 @@ +- Which framework has been used and why +- How to extend to more api diff --git a/docs/testing/developer/devguide/arch.rst b/docs/testing/developer/devguide/arch.rst new file mode 100644 index 00000000..d95faba6 --- /dev/null +++ b/docs/testing/developer/devguide/arch.rst @@ -0,0 +1,24 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2017 ZTE Corp. + + +######################## +QTIP Architecture Design +######################## + +In Danube, QTIP releases its standalone mode, which is also know as ``solo``: + +.. figure:: _assets/standalone.png + :alt: QTIP standalone mode + +The runner could be launched from CLI (command line interpreter) or API +(application programming interface) and drives the testing jobs. The generated +data including raw performance data and testing environment are fed to collector. +Performance metrics will be parsed from the raw data and used for QPI calculation. +Then the benchmark report is rendered with the benchmarking results. + +The execution can be detailed in the diagram below: + +.. figure:: _assets/sequence.png + :alt: QTIP execution sequence diff --git a/docs/testing/developer/design/cli.rst b/docs/testing/developer/devguide/cli.rst index 72d1fbaf..72d1fbaf 100644 --- a/docs/testing/developer/design/cli.rst +++ b/docs/testing/developer/devguide/cli.rst diff --git a/docs/testing/developer/design/compute_QPI.rst b/docs/testing/developer/devguide/compute-qpi.rst index 2e5aa87c..2e5aa87c 100644 --- a/docs/testing/developer/design/compute_QPI.rst +++ b/docs/testing/developer/devguide/compute-qpi.rst diff --git a/docs/testing/developer/design/index.rst b/docs/testing/developer/devguide/index.rst index b6dd0c01..2b4bd9b0 100644 --- a/docs/testing/developer/design/index.rst +++ b/docs/testing/developer/devguide/index.rst @@ -10,6 +10,7 @@ QTIP Design Specifications .. toctree:: :maxdepth: 2 - dashboard.rst - compute_QPI.rst - integration_with_yardstick.rst + arch.rst + cli.rst + api.rst + compute-qpi.rst diff --git a/docs/testing/user/.gitkeep b/docs/testing/user/.gitkeep deleted file mode 100644 index e69de29b..00000000 --- a/docs/testing/user/.gitkeep +++ /dev/null diff --git a/docs/testing/user/installation/.gitkeep b/docs/testing/user/installation/.gitkeep deleted file mode 100644 index e69de29b..00000000 --- a/docs/testing/user/installation/.gitkeep +++ /dev/null diff --git a/docs/testing/user/installation/README b/docs/testing/user/installation/README new file mode 100644 index 00000000..037d8dfa --- /dev/null +++ b/docs/testing/user/installation/README @@ -0,0 +1 @@ +Not applicable for non-installer project
\ No newline at end of file diff --git a/docs/testing/user/scenarios/.gitkeep b/docs/testing/user/scenarios/.gitkeep deleted file mode 100644 index e69de29b..00000000 --- a/docs/testing/user/scenarios/.gitkeep +++ /dev/null diff --git a/docs/testing/user/scenarios/README b/docs/testing/user/scenarios/README new file mode 100644 index 00000000..037d8dfa --- /dev/null +++ b/docs/testing/user/scenarios/README @@ -0,0 +1 @@ +Not applicable for non-installer project
\ No newline at end of file diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index 4be3e498..78c5d117 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -11,6 +11,5 @@ QTIP User Guide .. toctree:: :maxdepth: 2 - introduction.rst - benchmark-suites.rst - annex.rst + cli.rst + qpi-compute.rst diff --git a/docs/testing/user/userguide/qpi-compute.rst b/docs/testing/user/userguide/qpi-compute.rst new file mode 100644 index 00000000..369240c9 --- /dev/null +++ b/docs/testing/user/userguide/qpi-compute.rst @@ -0,0 +1,104 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + + +Compute Suite +============= + +Introduction +------------ + +The QTIP testing suite aims to benchmark the compute components of an OPNFV platform. +Such components include, the CPU performance, the memory performance. +Additionally virtual computing performance provided by the Hypervisor (KVM) installed as part of OPNFV platforms would be benchmarked too. + +The test suite consists of both synthetic and application specific benchmarks to test compute components. + +All the compute benchmarks could be run in 2 scenarios: + +1. On Baremetal Machines provisioned by an OPNFV installer (Host machines) +2. On Virtual Machines brought up through OpenStack on an OPNFV platform + +Note: The Compute benchmank suite constains relatively old benchmarks such as dhrystone and whetstone. The suite would be updated for better benchmarks such as Linbench for the OPNFV C release. + +Benchmarks +---------- + +The benchmarks include: + +Dhrystone 2.1 +^^^^^^^^^^^^^ + +Dhrystone is a synthetic benchmark for measuring CPU performance. It uses integer calculations to evaluate CPU capabilities. +Both Single CPU performance is measured along multi-cpu performance. + + +Dhrystone, however, is a dated benchmark and has some short comings. +Written in C, it is a small program that doesn't test the CPU memory subsystem. +Additionally, dhrystone results could be modified by optimizing the compiler and insome cases hardware configuration. + +References: http://www.eembc.org/techlit/datasheets/dhrystone_wp.pdf + +Whetstone +^^^^^^^^^ + +Whetstone is a synthetic benchmark to measure CPU floating point operation performance. +Both Single CPU performance is measured along multi-cpu performance. + +Like Dhrystone, Whetstone is a dated benchmark and has short comings. + +References: + +http://www.netlib.org/benchmark/whetstone.c + +OpenSSL Speed +^^^^^^^^^^^^^ + +OpenSSL Speed can be used to benchmark compute performance of a machine. In QTIP, two OpenSSL Speed benchmarks are incorporated: +1. RSA signatunes/sec signed by a machine +2. AES 128-bit encryption throughput for a machine for cipher block sizes + +References: + +https://www.openssl.org/docs/manmaster/apps/speed.html + +RAMSpeed +^^^^^^^^ + +RAMSpeed is used to measure a machine's memory perfomace. +The problem(array)size is large enough to ensure Cache Misses so that the main machine memory is used. +INTmem and FLOATmem benchmarks are executed in 4 different scenarios: + +a. Copy: a(i)=b(i) +b. Add: a(i)=b(i)+c(i) +c. Scale: a(i)=b(i)*d +d. Tniad: a(i)=b(i)+c(i)*d + +INTmem uses integers in these four benchmarks whereas FLOATmem uses floating points for these benchmarks. + +References: + +http://alasir.com/software/ramspeed/ + +https://www.ibm.com/developerworks/community/wikis/home?lang=en#!/wiki/W51a7ffcf4dfd_4b40_9d82_446ebc23c550/page/Untangling+memory+access+measurements + +DPI +^^^ + +nDPI is a modified variant of OpenDPI, Open source Deep packet Inspection, that is maintained by ntop. +An example application called *pcapreader* has been developed and is available for use along nDPI. + +A sample .pcap file is passed to the *pcapreader* application. +nDPI classifies traffic in the pcap file into different categories based on string matching. +The *pcapreader* application provides a throughput number for the rate at which traffic was classified, indicating a machine's computational performance. +The results are run 10 times and an average is taken for the obtained number. + +*nDPI may provide non consistent results and was added to Brahmaputra for experimental purposes* + +References: + +http://www.ntop.org/products/deep-packet-inspection/ndpi/ + +http://www.ntop.org/wp-content/uploads/2013/12/nDPI_QuickStartGuide.pdf diff --git a/docs/testing/user/userguide/_01-compute.rst b/legacy/docs/_01-compute.rst index 56be5488..56be5488 100644 --- a/docs/testing/user/userguide/_01-compute.rst +++ b/legacy/docs/_01-compute.rst diff --git a/docs/testing/user/userguide/_02-network.rst b/legacy/docs/_02-network.rst index 00fe5b0a..00fe5b0a 100644 --- a/docs/testing/user/userguide/_02-network.rst +++ b/legacy/docs/_02-network.rst diff --git a/docs/testing/user/userguide/_03-storage.rst b/legacy/docs/_03-storage.rst index b1490432..b1490432 100644 --- a/docs/testing/user/userguide/_03-storage.rst +++ b/legacy/docs/_03-storage.rst diff --git a/docs/testing/user/userguide/_testcase_description.rst b/legacy/docs/_testcase_description.rst index d60ca949..d60ca949 100644 --- a/docs/testing/user/userguide/_testcase_description.rst +++ b/legacy/docs/_testcase_description.rst diff --git a/docs/testing/user/userguide/annex.rst b/legacy/docs/annex.rst index e8bf5555..e8bf5555 100644 --- a/docs/testing/user/userguide/annex.rst +++ b/legacy/docs/annex.rst diff --git a/docs/testing/developer/design/apidocs/qtip_restful_api.rst b/legacy/docs/apidocs/qtip_restful_api.rst index 06c01292..06c01292 100644 --- a/docs/testing/developer/design/apidocs/qtip_restful_api.rst +++ b/legacy/docs/apidocs/qtip_restful_api.rst diff --git a/docs/testing/user/userguide/benchmark-suites.rst b/legacy/docs/benchmark-suites.rst index 84d1c647..84d1c647 100644 --- a/docs/testing/user/userguide/benchmark-suites.rst +++ b/legacy/docs/benchmark-suites.rst diff --git a/docs/testing/developer/design/apidocs/index.rst b/legacy/docs/index.rst index 241a2680..241a2680 100644 --- a/docs/testing/developer/design/apidocs/index.rst +++ b/legacy/docs/index.rst diff --git a/docs/testing/user/userguide/introduction.rst b/legacy/docs/introduction.rst index 3147f0aa..3147f0aa 100644 --- a/docs/testing/user/userguide/introduction.rst +++ b/legacy/docs/introduction.rst |
