diff options
| -rw-r--r-- | docs/release/release-notes.rst | 384 | ||||
| -rwxr-xr-x | docs/userguide/01-introduction.rst | 20 | ||||
| -rw-r--r-- | docs/userguide/02-methodology.rst | 19 | ||||
| -rwxr-xr-x | docs/userguide/03-architecture.rst | 7 | ||||
| -rw-r--r-- | docs/userguide/07-installation.rst | 107 | ||||
| -rw-r--r-- | docs/userguide/08-yardstick_plugin.rst | 8 | ||||
| -rw-r--r-- | docs/userguide/09-result-store-InfluxDB.rst | 4 | ||||
| -rw-r--r-- | docs/userguide/10-list-of-tcs.rst | 20 | ||||
| -rw-r--r-- | docs/userguide/opnfv_yardstick_tc053.rst | 2 | ||||
| -rw-r--r-- | docs/userguide/opnfv_yardstick_tc074.rst | 4 | ||||
| -rw-r--r-- | docs/userguide/references.rst | 12 | ||||
| -rwxr-xr-x | tests/ci/prepare_env.sh | 14 | ||||
| -rwxr-xr-x | tests/ci/yardstick-verify | 51 | ||||
| -rwxr-xr-x | tools/yardstick-img-modify | 2 |
14 files changed, 388 insertions, 266 deletions
diff --git a/docs/release/release-notes.rst b/docs/release/release-notes.rst index bc58b2134..c4c7ecf65 100644 --- a/docs/release/release-notes.rst +++ b/docs/release/release-notes.rst @@ -5,7 +5,7 @@ ============================================ -OPNFV Brahmaputra Release Note for Yardstick +OPNFV Colorado Release Note for Yardstick ============================================ .. toctree:: @@ -21,12 +21,17 @@ OPNFV Brahmaputra Release Note for Yardstick Abstract ======== -This document compiles the release notes for the OPNFV Brahmaputra release -for Yardstick framework as well as Yardstick_ Project deliverables. +This document describes the release note of Yardstick project. + License ======= +OPNFV Colorado release note for Yardstick Docs +are licensed under a Creative Commons Attribution 4.0 International License. +You should have received a copy of the license along with this. +If not, see <http://creativecommons.org/licenses/by/4.0/>. + The *Yardstick framework*, the *Yardstick test cases* and the *ApexLake* experimental framework are opensource software, licensed under the terms of the Apache License, Version 2.0. @@ -35,19 +40,13 @@ Apache License, Version 2.0. Version History =============== -+---------------+--------------------+---------------------------------+ -| *Date* | *Version* | *Comment* | -| | | | -+---------------+--------------------+---------------------------------+ -| Apr 27th,2016 | 3.0 | Brahmaputra release | -| | | | -+---------------+--------------------+---------------------------------+ -| Mar 30th,2016 | 2.0 | Brahmaputra release | -| | | | -+---------------+--------------------+---------------------------------+ -| Feb 25th,2016 | 1.0 | Brahmaputra release | -| | | | -+---------------+--------------------+---------------------------------+ ++----------------+--------------------+---------------------------------+ +| *Date* | *Version* | *Comment* | +| | | | ++----------------+--------------------+---------------------------------+ +| Aug 22nd, 2016 | 1.0 | Yardstick for Colorado release | +| | | | ++----------------+--------------------+---------------------------------+ Important Notes @@ -65,7 +64,7 @@ independent. Summary ======= -This Brahmaputra release provides *Yardstick* as a framework for NFVI testing +This Colorado release provides *Yardstick* as a framework for NFVI testing and OPNFV feature testing, automated in the OPNFV CI pipeline, including: * Documentation generated with Sphinx @@ -84,14 +83,16 @@ and OPNFV feature testing, automated in the OPNFV CI pipeline, including: * Automated Yardstick test results visualization - * Dashboard_ using Grafana (user:opnfv/password: opnfv), influxDB used as + * Dashboard_ using Grafana (user:opnfv/password: opnfv), influxDB is used as backend * Yardstick framework source code * Yardstick test cases yaml files -For Brahmaputra release, the *Yardstick framework* is used for the following +* Yardstick pliug-in configration yaml files, plug-in install/remove scripts + +For Colorado release, the *Yardstick framework* is used for the following testing: * OPNFV platform testing - generic test cases to measure the categories: @@ -112,7 +113,9 @@ testing: * Parser -* Test cases added in Brahmaputra2.0: + * StorPerf + + * VSperf * virtual Traffic Classifier @@ -132,165 +135,52 @@ Release Data | **Project** | Yardstick | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | yardstick/brahmaputra.3.0 | +| **Repo/tag** | yardstick/colorado.1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Yardstick Docker image tag** | brahmaputra.3.0 | +| **Yardstick Docker image tag** | colorado.1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Brahmaputra | +| **Release designation** | Colorado base release | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | Apr 27th, 2016 | +| **Release date** | September 22 2016 | | | | +--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | OPNFV Brahmaputra release | +| **Purpose of the delivery** | Colorado base release | | | | +--------------------------------------+--------------------------------------+ -Version Change --------------- - -Module Version Changes -~~~~~~~~~~~~~~~~~~~~~~ - -This is the third tracked release of Yardstick. It is based on following -upstream versions: - -- OpenStack Liberty - -- OpenDaylight Beryllium - - -Document Version Changes -~~~~~~~~~~~~~~~~~~~~~~~~ - -This is the third tracked version of the Yardstick framework in OPNFV. -It includes the following documentation updates: - -- Yardstick User Guide: corrected faulty links - -- Yardstick Code Documentation: no changes - -- Yardstick Release Notes for Yardstick: this document - -- Test Results report for Brahmaputra testing with Yardstick: updated listed of -verified scenarios and limitations - -Documentation updates on the second tracked version: - -- Yardstick User Guide: added software architecture chapter - -- Yardstick Code Documentation: no changes - -- Yardstick Release Notes for Yardstick: this document - -- Test Results report for Brahmaputra testing with Yardstick: added test cases -and results for virtual Traffic Classifier - - -Reason for Version ------------------- - -Feature additions -~~~~~~~~~~~~~~~~~ - -No new features. - -Brahmaputra.2.0: - -+----------------------------+------------------------------------------------+ -| **JIRA REFERENCE** | **SLOGAN** | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-227 | Heat HTTPS SSL support. | -| | | -+----------------------------+------------------------------------------------+ - - -Corrected Faults -~~~~~~~~~~~~~~~~ - -No corrected faults. - -Brahmaputra.2.0: - -+----------------------------+------------------------------------------------+ -| **JIRA REFERENCE** | **SLOGAN** | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: - | Change copyrights for base scenario, runners, | -| | dispatchers, cover. | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: - | Update setup.py and dependencies | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: - | Add missing dependencies to docker file | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: - | Fix Heat template for noisy neighbors deploy | -| | | -+----------------------------+------------------------------------------------+ - -Known Faults -~~~~~~~~~~~~ - - -+----------------------------+------------------------------------------------+ -| **JIRA REFERENCE** | **SLOGAN** | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-175 | Running test suite, if a test cases running | -| | failed, the test is stopped. | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-176 | Fix plotter bug since Output format has been | -| | changed. | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-216 | ArgsAlreadyParsedError: arguments already | -| | parsed: cannot register CLI option. | -| | | -+----------------------------+------------------------------------------------+ -| JIRA: YARDSTICK-231 | Installation instructions on Wiki not accurate | -| | | -+----------------------------+------------------------------------------------+ - -.. note:: The faults not related to *Yardstick* framework, addressing scenarios - which were not fully verified, are listed in the OPNFV installer's release - notes. - - Deliverables ------------- +============ Software Deliverables -~~~~~~~~~~~~~~~~~~~~~ +--------------------- -**Yardstick framework source code <brahmaputra.3.0>** +**Yardstick framework source code <colorado.1.0>** +--------------------------------------+--------------------------------------+ | **Project** | Yardstick | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | yardstick/brahmaputra.3.0 | +| **Repo/tag** | yardstick/colorado.1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Yardstick Docker image tag** | brahmaputra.3.0 | +| **Yardstick Docker image tag** | colorado.1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Brahmaputra | +| **Release designation** | colorado | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | Apr 27th, 2016 | +| **Release date** | September 22th, 2016 | | | | +--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | OPNFV Brahmaputra release | +| **Purpose of the delivery** | OPNFV Colorado release | | | | +--------------------------------------+--------------------------------------+ + **Contexts** +---------------------+-------------------------------------------------------+ @@ -326,6 +216,7 @@ Software Deliverables | | | +---------------------+-------------------------------------------------------+ + **Scenarios** +---------------------+-------------------------------------------------------+ @@ -353,15 +244,27 @@ Software Deliverables | | | | | * lmbench | | | | +| | * lmbench_cache | +| | | | | * perf | | | | | | * unixbench | | | | +| | * ramspeed | +| | | +| | * cachestat | +| | | +| | * memeoryload | +| | | +| | * computecapacity | +| | | +---------------------+-------------------------------------------------------+ | *Networking* | * iperf3 | | | | | | * netperf | | | | +| | * netperf_node | +| | | | | * ping | | | | | | * ping6 | @@ -380,13 +283,23 @@ Software Deliverables | | | | | * vtc throughput in the presence of noisy neighbors | | | | +| | * networkcapacity | +| | | +| | * netutilization | +| | | +---------------------+-------------------------------------------------------+ | *Parser* | Tosca2Heat | | | | +---------------------+-------------------------------------------------------+ | *Storage* | fio | | | | +| | storagecapacity | +| | | +---------------------+-------------------------------------------------------+ +| *StorPerf* | storperf | +| | | ++---------------------+-------------------------------------------------------+ + **API to Other Frameworks** @@ -401,6 +314,7 @@ Software Deliverables | | | +---------------------+-------------------------------------------------------+ + **Test Results Output** +-----------------------------+-----------------------------------------------+ @@ -413,13 +327,13 @@ Software Deliverables | http | Post data to html. | | | | +-----------------------------+-----------------------------------------------+ -| influxdb | Post data to influxdB. | +| influxdb | Post data to influxDB. | | | | +-----------------------------+-----------------------------------------------+ Delivered Test cases -~~~~~~~~~~~~~~~~~~~~ +-------------------- * Generic NFVI test cases @@ -427,6 +341,8 @@ Delivered Test cases * OPNFV_YARDSTICK_TCOO2 - NW Latency + * OPNFV_YARDSTICK_TCOO4 - Cache Utilization + * OPNFV_YARDSTICK_TCOO5 - Storage Performance * OPNFV_YARDSTICK_TCOO8 - Packet Loss Extended Test @@ -448,6 +364,31 @@ Delivered Test cases * OPNFV_YARDSTICK_TCO38 - Latency, CPU Load, Throughput, Packet Loss Extended Test + * OPNFV_YARDSTICK_TCO42 - Network Performance + + * OPNFV_YARDSTICK_TCO43 - Network Latency Between NFVI Nodes + + * OPNFV_YARDSTICK_TCO44 - Memory Utilization + + * OPNFV_YARDSTICK_TCO55 - Compute Capacity + + * OPNFV_YARDSTICK_TCO61 - Network Utilization + + * OPNFV_YARDSTICK_TCO63 - Storage Capacity + + * OPNFV_YARDSTICK_TCO69 - Memory Bandwidth + + * OPNFV_YARDSTICK_TCO70 - Latency, Memory Utilization, Throughput, Packet + Loss + + * OPNFV_YARDSTICK_TCO71 - Latency, Cache Utilization, Throughput, Packet Loss + + * OPNFV_YARDSTICK_TCO72 - Latency, Network Utilization, Throughput, Packet + Loss + + * OPNFV_YARDSTICK_TC073 - Network Latency and Throughput Between Nodes + + * OPNFV_YARDSTICK_TCO75 - Network Capacity and Scale * Test Cases for OPNFV HA Project: @@ -455,6 +396,34 @@ Delivered Test cases * OPNFV_YARDSTICK_TC025 - HA: OpenStacK Controller Node abnormally down + * OPNFV_YARDSTICK_TCO45 - HA: Control node Openstack service down - neutron + server + + * OPNFV_YARDSTICK_TC046 - HA: Control node Openstack service down - keystone + + * OPNFV_YARDSTICK_TCO47 - HA: Control node Openstack service down - glance + api + + * OPNFV_YARDSTICK_TC048 - HA: Control node Openstack service down - cinder + api + + * OPNFV_YARDSTICK_TCO49 - HA: Control node Openstack service down - swift + proxy + + * OPNFV_YARDSTICK_TC050 - HA: OpenStack Controller Node Network High + Availability + + * OPNFV_YARDSTICK_TCO51 - HA: OpenStack Controller Node CPU Overload High + Availability + + * OPNFV_YARDSTICK_TC052 - HA: OpenStack Controller Node Disk I/O Block High + Availability + + * OPNFV_YARDSTICK_TCO53 - HA: OpenStack Controller Load Balance Service High + Availability + + * OPNFV_YARDSTICK_TC054 - HA: OpenStack Virtual IP High Availability + * Test Case for OPNFV IPv6 Project: * OPNFV_YARDSTICK_TCO27 - IPv6 connectivity @@ -467,6 +436,10 @@ Delivered Test cases * OPNFV_YARDSTICK_TCO40 - Verify Parser Yang-to-Tosca +* Test Case for OPNFV StorPerf Project: + + * OPNFV_YARDSTICK_TCO74 - Storperf + * Test Cases for Virtual Traffic Classifier: * OPNFV_YARDSTICK_TC006 - Virtual Traffic Classifier Data Plane Throughput @@ -479,3 +452,116 @@ Benchmarking in presence of noisy neighbors Test * OPNFV_YARDSTICK_TC021 - Virtual Traffic Classifier Instantiation in presence of noisy neighbors Test + + +Version Change +============== + +Module Version Changes +---------------------- + +This is the third tracked release of Yardstick. It is based on following +upstream versions: + +- OpenStack Mitaka + +- OpenDaylight Beryllium + + +Document Version Changes +------------------------ + +This is the third tracked version of the Yardstick framework in OPNFV. +It includes the following documentation updates: + +- Yardstick User Guide: corrected faulty links + +- Yardstick Code Documentation: no changes + +- Yardstick Release Notes for Yardstick: this document + +- Test Results report for Colorado testing with Yardstick: updated listed of +verified scenarios and limitations + +Documentation updates on the second tracked version: + +- Yardstick User Guide: added software architecture chapter + +- Yardstick Code Documentation: no changes + +- Yardstick Release Notes for Yardstick: this document + +- Test Results report for Brahmaputra testing with Yardstick: added test cases +and results for virtual Traffic Classifier + + +Reason for Version +================== +* TODO * + +Feature additions +----------------- +* TODO * + +Corrected Faults +---------------- +* TODO * + +Known Issues/Faults +------------ +* TODO * + +.. note:: The faults not related to *Yardstick* framework, addressing scenarios + which were not fully verified, are listed in the OPNFV installer's release + notes. + + +Colorado known restrictions/issues +================================== ++-----------+-----------+----------------------------------------------+ +| Installer | Scenario | Issue | ++===========+===========+==============================================+ +| any | *-bgpvpn | floating ips not supported. Some Test cases | +| | | related to floating ips are excluded. | ++-----------+-----------+----------------------------------------------+ +* TODO * + + +Test results +============ + +Test results are available in: + + - jenkins logs on CI: https://build.opnfv.org/ci/view/yardstick/ + + +Open JIRA tickets +================= + ++------------------+-----------------------------------------------+ +| JIRA | Description | ++==================+===============================================+ ++------------------+-----------------------------------------------+ ++------------------+-----------------------------------------------+ ++------------------+-----------------------------------------------+ ++------------------+-----------------------------------------------+ ++------------------+-----------------------------------------------+ + + +Useful links +============ + + - wiki project page: https://wiki.opnfv.org/display/yardstick/Yardstick + + - wiki Yardstick Colorado release planing page: https://wiki.opnfv.org/display/yardstick/Yardstick+Colorado+Release+Planning + + - wiki Yardstick Colorado release jira page: https://wiki.opnfv.org/display/yardstick/Jira+Yardstick-Colorado + + - Yardstick repo: https://git.opnfv.org/cgit/yardstick + + - Yardstick CI dashboard: https://build.opnfv.org/ci/view/yardstick + + - Yardstick grafana dashboard: http://testresults.opnfv.org/grafana/ + + - Yardstick IRC chanel: #opnfv-yardstick + diff --git a/docs/userguide/01-introduction.rst b/docs/userguide/01-introduction.rst index 3db6ce001..9d9cf0fb5 100755 --- a/docs/userguide/01-introduction.rst +++ b/docs/userguide/01-introduction.rst @@ -40,18 +40,25 @@ This document consists of the following chapters: * Chapter :doc:`02-methodology` describes the methodology implemented by the Yardstick Project for :term:`NFVI` verification. -* Chapter :doc:`architecture` provides information on the software architecture +* Chapter :doc:`03-architecture` provides information on the software architecture of yardstick. + * Chapter :doc:`04-vtc-overview` provides information on the :term:`VTC`. -* Chapter :doc:`apexlake_installation` provides instructions to install the - experimental framework *ApexLake* and chapter :doc:`apexlake_api` explains +* Chapter :doc:`05-apexlake_installation` provides instructions to install the + experimental framework *ApexLake* and chapter :doc:`06-apexlake_api` explains how this framework is integrated in *Yardstick*. -* Chapter :doc:`03-installation` provides instructions to install *Yardstick*. +* Chapter :doc:`07-installation` provides instructions to install *Yardstick*. + +* Chapter :doc:`08-yardstick_plugin` provides information on how to integrate + other OPNFV testing projects into *Yardstick*. -* Chapter :doc:`03-list-of-tcs` includes a list of available Yardstick - test cases. +* Chapter :doc:`09-result-store-InfluxDB` provides inforamtion on how to run + plug-in test cases and store test results into community's InfluxDB. + +* Chapter :doc:`10-list-of-tcs` includes a list of available Yardstick test + cases. Contact Yardstick @@ -60,3 +67,4 @@ Contact Yardstick Feedback? `Contact us`_ .. _Contact us: opnfv-users@lists.opnfv.org + diff --git a/docs/userguide/02-methodology.rst b/docs/userguide/02-methodology.rst index 3fa432a98..1849ffd0c 100644 --- a/docs/userguide/02-methodology.rst +++ b/docs/userguide/02-methodology.rst @@ -59,7 +59,7 @@ The metrics, as defined by ETSI GS NFV-TST001, are shown in :ref:`Table1 <table2_1>`, :ref:`Table2 <table2_2>` and :ref:`Table3 <table2_3>`. -In OPNFV Brahmaputra release, generic test cases covering aspects of the listed +In OPNFV Colorado release, generic test cases covering aspects of the listed metrics are available; further OPNFV releases will provide extended testing of these metrics. The view of available Yardstick test cases cross ETSI definitions in @@ -169,24 +169,25 @@ options). | | | | | +---------+-------------------+----------------+------------------------------+ | Compute | TC003 [1]_ | TC003 [1]_ | TC013 [1]_ | -| | TC004 [1]_ | TC004 [1]_ | TC015 [1]_ | +| | TC004 | TC004 | TC015 [1]_ | | | TC014 | TC010 | | | | TC024 | TC012 | | -| | | | | +| | TC069 | TC055 | | +---------+-------------------+----------------+------------------------------+ -| Network | TC002 | TC001 | TC016 [1]_ | -| | TC011 | TC008 | TC018 [1]_ | -| | | TC009 | | -| | | | | +| Network | TC001 | TC001 | TC016 [1]_ | +| | TC002 | TC008 | TC018 [1]_ | +| | TC011 | TC009 | | +| | TC073 | TC075 | | +---------+-------------------+----------------+------------------------------+ | Storage | TC005 | TC005 | TC017 [1]_ | -| | | | | +| | | TC063 | | +---------+-------------------+----------------+------------------------------+ .. note:: The description in this OPNFV document is intended as a reference for users to understand the scope of the Yardstick Project and the deliverables of the Yardstick framework. For complete description of - the methodology, refer to the ETSI document. + the methodology, please refer to the ETSI document. .. rubric:: Footnotes .. [1] To be included in future deliveries. + diff --git a/docs/userguide/03-architecture.rst b/docs/userguide/03-architecture.rst index 3abb67b7d..ace3117c2 100755 --- a/docs/userguide/03-architecture.rst +++ b/docs/userguide/03-architecture.rst @@ -222,7 +222,7 @@ Deployment View =============== Yardstick deployment view shows how the yardstick tool can be deployed into the underlying platform. Generally, yardstick tool is installed on JumpServer(see -`03-installation` for detail installation steps), and JumpServer is +`07-installation` for detail installation steps), and JumpServer is connected with other control/compute servers by networking. Based on this deployment, yardstick can run the test cases on these hosts, and get the test result for better showing. @@ -256,8 +256,11 @@ Yardstick Directory structure by Heat. Currently contains how to build the yardstick-trusty-server image with the different tools that are needed from within the image. +*plugin/* - Plug-in configuration files are stored here. + *vTC/* - Contains the files for running the virtual Traffic Classifier tests. *yardstick/* - Contains the internals of Yardstick: Runners, Scenario, Contexts, - CLI parsing, keys, plotting tools, dispatcher and so on. + CLI parsing, keys, plotting tools, dispatcher, plugin + install/remove scripts and so on. diff --git a/docs/userguide/07-installation.rst b/docs/userguide/07-installation.rst index 475719c72..3a0a7975f 100644 --- a/docs/userguide/07-installation.rst +++ b/docs/userguide/07-installation.rst @@ -29,9 +29,21 @@ Installing Yardstick on Ubuntu 14.04 .. _install-framework: +You can install Yardstick framework directly on Ubuntu 14.04 or in an Ubuntu +14.04 Docker image. +No matter which way you choose to install Yardstick framework, the following +installation steps are identical. +If you choose to use the Ubuntu 14.04 Docker image, You can pull the Ubuntu +14.04 Docker image from Docker hub: + +:: + + docker pull ubuntu:14.04 + Installing Yardstick framework ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Install dependencies: + :: sudo apt-get update && sudo apt-get install -y \ @@ -50,6 +62,7 @@ Install dependencies: python-setuptools Create a python virtual environment, source it and update setuptools: + :: virtualenv ~/yardstick_venv @@ -57,6 +70,7 @@ Create a python virtual environment, source it and update setuptools: easy_install -U setuptools Download source code and install python dependencies: + :: git clone https://gerrit.opnfv.org/gerrit/yardstick @@ -69,17 +83,6 @@ There is also a YouTube video, showing the above steps: :alt: http://www.youtube.com/watch?v=4S4izNolmR0 :target: http://www.youtube.com/watch?v=4S4izNolmR0 -Installing extra tools -^^^^^^^^^^^^^^^^^^^^^^ -yardstick-plot -"""""""""""""" -Yardstick has an internal plotting tool ``yardstick-plot``, which can be installed -using the following command: -:: - - sudo apt-get install -y g++ libfreetype6-dev libpng-dev pkg-config - python setup.py develop easy_install yardstick[plot] - .. _guest-image: Building a guest image @@ -90,6 +93,7 @@ have sudo rights to use this tool. Also you may need install several additional packages to use this tool, by follwing the commands below: + :: apt-get update && apt-get install -y \ @@ -99,6 +103,7 @@ follwing the commands below: This image can be built using the following command while in the directory where Yardstick is installed (``~/yardstick`` if the framework is installed by following the commands above): + :: sudo ./tools/yardstick-img-modify tools/ubuntu-server-cloudimg-modify.sh @@ -110,6 +115,7 @@ The created image can be added to OpenStack using the ``glance image-create`` or via the OpenStack Dashboard. Example command: + :: glance --os-image-api-version 1 image-create \ @@ -121,44 +127,18 @@ Example command: Installing Yardstick using Docker --------------------------------- -Yardstick has two Docker images, first one (**Yardstick-framework**) serves as a -replacement for installing the Yardstick framework in a virtual environment (for -example as done in :ref:`install-framework`), while the other image is mostly for -CI purposes (**Yardstick-CI**). - -Yardstick-framework image -^^^^^^^^^^^^^^^^^^^^^^^^^ -Download the source code: - -:: - - git clone https://gerrit.opnfv.org/gerrit/yardstick - -Build the Docker image and tag it as *yardstick-framework*: - -:: - - cd yardstick - docker build -t yardstick-framework . - -Run the Docker instance: - -:: - - docker run --name yardstick_instance -i -t yardstick-framework +Yardstick iteself has a Docker image, this Docker image (**Yardstick-stable**) +serves as a replacement for installing the Yardstick framework in a virtual +environment (for example as done in :ref:`install-framework`). +It is recommended to use this Docker image to run Yardstick test. -To build a guest image for Yardstick, see :ref:`guest-image`. - -Yardstick-CI image -^^^^^^^^^^^^^^^^^^ -Pull the Yardstick-CI Docker image from Docker hub: +Yardstick-stable image +^^^^^^^^^^^^^^^^^^^^^^ +Pull the Yardstick-stable Docker image from Docker hub: :: - docker pull opnfv/yardstick:$DOCKER_TAG - -Where ``$DOCKER_TAG`` is latest for master branch, as for the release branches, -this coincides with its release name, such as brahmaputra.1.0. + docker pull opnfv/yardstick:stable Run the Docker image: @@ -180,21 +160,32 @@ For more details, please refer to the Jenkins job defined in Releng project, lab and sshkey are required. See the link https://git.opnfv.org/cgit/releng/tree/jjb/yardstick/yardstick-ci-jobs.yml. -Note: exec_tests.sh is used for executing test suite here, furthermore, if someone wants to execute the -test suite manually, it can be used as long as the parameters are configured correct. Another script -called run_tests.sh is used for unittest in Jenkins verify job, in local manaul environment, -it is recommended to run before test suite execuation. +Note: exec_tests.sh is used for executing test suite here, furthermore, if someone +wants to execute the test suite manually, it can be used as long as the parameters +are configured correct. Another script called run_tests.sh is used for unittest in +Jenkins verify job, in local manaul environment, it is recommended to run before +test suite execuation. -Basic steps performed by the **Yardstick-CI** container: +Basic steps performed by the **Yardstick-stable** container: 1. clone yardstick and releng repos 2. setup OS credentials (releng scripts) 3. install yardstick and dependencies 4. build yardstick cloud image and upload it to glance -5. upload cirros-0.3.3 cloud image to glance +5. upload cirros-0.3.3 cloud image and ubuntu-14.04 cloud image to glance 6. run yardstick test scenarios 7. cleanup +If someone only wants to execute a single test case, one can log into the yardstick-stable +container first using command: + +:: + + docker run -it openfv/yardstick /bin/bash + +Then in the container run yardstick task command to execute single test case. +Detailed steps about executing Yardstick test case can be found below. + OpenStack parameters and credentials ------------------------------------ @@ -240,18 +231,8 @@ of yardstick help command and ping.py test sample: Each testing tool supported by Yardstick has a sample configuration file. These configuration files can be found in the **samples** directory. -Example invocation of ``yardstick-plot`` tool: -:: - - yardstick-plot -i /tmp/yardstick.out -o /tmp/plots/ - Default location for the output is ``/tmp/yardstick.out``. -More info about the tool can be found by executing: -:: - - yardstick-plot -h - Deploy InfluxDB and Grafana locally ------------------------------------ @@ -259,6 +240,7 @@ Deploy InfluxDB and Grafana locally .. pull docker images Pull docker images + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: @@ -299,6 +281,8 @@ Config grafana log on using admin/admin and config database resource to be {YOUR_IP_HERE}:8086 .. image:: images/Grafana_config.png + :width: 800px + :alt: Grafana data source configration Config yardstick conf ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -388,3 +372,4 @@ the ci environment. "task_args" is where you can specify the task arguments for All in all, to create a test suite in yardstick, you just need to create a suite yaml file and add test cases and constraint or task arguments if necessary. + diff --git a/docs/userguide/08-yardstick_plugin.rst b/docs/userguide/08-yardstick_plugin.rst index e68db650d..f16dedd02 100644 --- a/docs/userguide/08-yardstick_plugin.rst +++ b/docs/userguide/08-yardstick_plugin.rst @@ -48,11 +48,11 @@ environment and other dependencies: 3. Make sure Jump Host have access to the OpenStack Controller API. 4. Make sure Jump Host must have internet connectivity for downloading docker image. 5. You need to know where to get basic openstack Keystone authorization info, such as -OS_PASSWORD, OS_TENANT_NAME, OS_AUTH_URL, OS_USERNAME. + OS_PASSWORD, OS_TENANT_NAME, OS_AUTH_URL, OS_USERNAME. 6. To run a Storperf container, you need to have OpenStack Controller environment -variables defined and passed to Storperf container. The best way to do this is to -put environment variables in a "storperf_admin-rc" file. The storperf_admin-rc -should include credential environment variables at least: + variables defined and passed to Storperf container. The best way to do this is to + put environment variables in a "storperf_admin-rc" file. The storperf_admin-rc + should include credential environment variables at least: * OS_AUTH_URL * OS_TENANT_ID diff --git a/docs/userguide/09-result-store-InfluxDB.rst b/docs/userguide/09-result-store-InfluxDB.rst index 5c49e9f7c..a0bb48a80 100644 --- a/docs/userguide/09-result-store-InfluxDB.rst +++ b/docs/userguide/09-result-store-InfluxDB.rst @@ -17,7 +17,7 @@ into community's InfluxDB. The framework is shown in Framework_. .. image:: images/InfluxDB_store.png - :width: 1200px + :width: 800px :alt: Store Other Project's Test Results in InfluxDB Store Storperf Test Results into Community's InfluxDB @@ -81,6 +81,6 @@ can be accessed by Login_. .. image:: images/results_visualization.png - :width: 1200px + :width: 800px :alt: results visualization diff --git a/docs/userguide/10-list-of-tcs.rst b/docs/userguide/10-list-of-tcs.rst index 7e8c85433..8798a8f51 100644 --- a/docs/userguide/10-list-of-tcs.rst +++ b/docs/userguide/10-list-of-tcs.rst @@ -49,6 +49,7 @@ Generic NFVI Test Case Descriptions opnfv_yardstick_tc070.rst opnfv_yardstick_tc071.rst opnfv_yardstick_tc072.rst + opnfv_yardstick_tc073.rst opnfv_yardstick_tc075.rst OPNFV Feature Test Cases @@ -62,6 +63,16 @@ H A opnfv_yardstick_tc019.rst opnfv_yardstick_tc025.rst + opnfv_yardstick_tc045.rst + opnfv_yardstick_tc046.rst + opnfv_yardstick_tc047.rst + opnfv_yardstick_tc048.rst + opnfv_yardstick_tc049.rst + opnfv_yardstick_tc050.rst + opnfv_yardstick_tc051.rst + opnfv_yardstick_tc052.rst + opnfv_yardstick_tc053.rst + opnfv_yardstick_tc054.rst IPv6 ---- @@ -87,6 +98,14 @@ Parser opnfv_yardstick_tc040.rst + StorPerf +----------- + +.. toctree:: + :maxdepth: 1 + + opnfv_yardstick_tc074.rst + virtual Traffic Classifier -------------------------- @@ -106,3 +125,4 @@ Templates testcase_description_v2_template Yardstick_task_templates + diff --git a/docs/userguide/opnfv_yardstick_tc053.rst b/docs/userguide/opnfv_yardstick_tc053.rst index 8808d12d9..3c6bbc628 100644 --- a/docs/userguide/opnfv_yardstick_tc053.rst +++ b/docs/userguide/opnfv_yardstick_tc053.rst @@ -13,7 +13,7 @@ Yardstick Test Case Description TC053 | | +--------------+--------------------------------------------------------------+ |test case id | OPNFV_YARDSTICK_TC053: OpenStack Controller Load Balance | -| | Service High Availability- | +| | Service High Availability | +--------------+--------------------------------------------------------------+ |test purpose | This test case will verify the high availability of the | | | load balance service(current is HAProxy) that supports | diff --git a/docs/userguide/opnfv_yardstick_tc074.rst b/docs/userguide/opnfv_yardstick_tc074.rst index c938f5dfd..92cd51439 100644 --- a/docs/userguide/opnfv_yardstick_tc074.rst +++ b/docs/userguide/opnfv_yardstick_tc074.rst @@ -7,7 +7,7 @@ Yardstick Test Case Description TC074 ************************************* -.. Storperf: https://wiki.opnfv.org/display/storperf/Storperf +.. _Storperf: https://wiki.opnfv.org/display/storperf/Storperf +-----------------------------------------------------------------------------+ |Storperf | @@ -44,7 +44,7 @@ Yardstick Test Case Description TC074 | | * timeout: 600 - maximum allowed job time | | | | +--------------+--------------------------------------------------------------+ -|test tool | Storperf | +|test tool | Storperf_ | | | | | | StorPerf is a tool to measure block and object storage | | | performance in an NFVI. | diff --git a/docs/userguide/references.rst b/docs/userguide/references.rst index 551926135..7f6a0f0f0 100644 --- a/docs/userguide/references.rst +++ b/docs/userguide/references.rst @@ -22,18 +22,27 @@ OPNFV References used in Test Cases ============================= +* cachestat: https://github.com/brendangregg/perf-tools/tree/master/fs * cirros-image: https://download.cirros-cloud.net * cyclictest: https://rt.wiki.kernel.org/index.php/Cyclictest * DPDKpktgen: https://github.com/Pktgen/Pktgen-DPDK/ * DPDK supported NICs: http://dpdk.org/doc/nics +* fdisk: http://www.tldp.org/HOWTO/Partition/fdisk_partitioning.html * fio: http://www.bluestop.org/fio/HOWTO.txt +* free: http://manpages.ubuntu.com/manpages/trusty/en/man1/free.1.html * iperf3: https://iperf.fr/ +* iostat: http://linux.die.net/man/1/iostat * Lmbench man-pages: http://manpages.ubuntu.com/manpages/trusty/lat_mem_rd.8.html * Memory bandwidth man-pages: http://manpages.ubuntu.com/manpages/trusty/bw_mem.8.html -* unixbench: https://github.com/kdlucas/byte-unixbench/blob/master/UnixBench * mpstat man-pages: http://manpages.ubuntu.com/manpages/trusty/man1/mpstat.1.html +* netperf: http://www.netperf.org/netperf/training/Netperf.html * pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt +* RAMspeed: http://alasir.com/software/ramspeed/ +* sar: http://linux.die.net/man/1/sar * SR-IOV: https://wiki.openstack.org/wiki/SR-IOV-Passthrough-For-Networking +* Storperf: https://wiki.opnfv.org/display/storperf/Storperf +* unixbench: https://github.com/kdlucas/byte-unixbench/blob/master/UnixBench + Research ======== @@ -48,3 +57,4 @@ Standards * ETSI NFV: http://www.etsi.org/technologies-clusters/technologies/nfv * ETSI GS-NFV TST 001: https://docbox.etsi.org/ISG/NFV/Open/Drafts/TST001_-_Pre-deployment_Validation/ * RFC2544: https://www.ietf.org/rfc/rfc2544.txt + diff --git a/tests/ci/prepare_env.sh b/tests/ci/prepare_env.sh index bbd7b5553..9c949ec4f 100755 --- a/tests/ci/prepare_env.sh +++ b/tests/ci/prepare_env.sh @@ -15,7 +15,6 @@ : ${NODE_NAME:='unknown'} : ${EXTERNAL_NETWORK:='admin_floating_net'} - # Extract network name from EXTERNAL_NETWORK # e.g. EXTERNAL_NETWORK='ext-net;flat;192.168.0.2;192.168.0.253;192.168.0.1;192.168.0.0/24' export EXTERNAL_NETWORK=$(echo $EXTERNAL_NETWORK | cut -f1 -d \;) @@ -75,6 +74,13 @@ verify_connectivity() { error "Can not talk to $ip." } +YARD_IMG_ARCH=amd64 +export YARD_IMG_ARCH + +if ! grep -q "Defaults env_keep += \"YARD_IMG_ARCH\"" "/etc/sudoers"; then + sudo echo "Defaults env_keep += \"YARD_IMG_ARCH YARDSTICK_REPO_DIR\"" >> /etc/sudoers +fi + ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" if [ "$INSTALLER_TYPE" == "fuel" ]; then @@ -85,13 +91,7 @@ if [ "$INSTALLER_TYPE" == "fuel" ]; then root@${INSTALLER_IP}:~/.ssh/id_rsa /root/.ssh/id_rsa &> /dev/null ARCH_SCRIPT="test -f /etc/fuel_openstack_arch && grep -q arm64 /etc/fuel_openstack_arch" - YARD_IMG_ARCH=amd64 sshpass -p r00tme ssh $ssh_options -l root $INSTALLER_IP "${ARCH_SCRIPT}" && YARD_IMG_ARCH=arm64 - export YARD_IMG_ARCH - - if ! grep -q "Defaults env_keep += \"YARD_IMG_ARCH\"" "/etc/sudoers"; then - sudo echo "Defaults env_keep += \"YARD_IMG_ARCH YARDSTICK_REPO_DIR\"" >> /etc/sudoers - fi sshpass -p r00tme ssh 2>/dev/null $ssh_options \ root@${INSTALLER_IP} fuel node>fuel_node diff --git a/tests/ci/yardstick-verify b/tests/ci/yardstick-verify index 8670bf4d0..37d0a9036 100755 --- a/tests/ci/yardstick-verify +++ b/tests/ci/yardstick-verify @@ -197,7 +197,7 @@ load_cirros_image() output=$(glance image-create \ --name cirros-0.3.3 \ - --disk-format qcow2 \ + --disk-format $DISK_FORMAT \ --container-format bare \ --file $image_file) echo "$output" @@ -270,12 +270,6 @@ load_yardstick_image() cd $YARDSTICK_REPO_DIR fi - if [ "$DEPLOY_SCENARIO" == "os-nosdn-lxd-ha" -o "$DEPLOY_SCENARIO" == "os-nosdn-lxd-noha" ]; then - DISK_FORMAT="raw" - else - DISK_FORMAT="qcow2" - fi - output=$(eval glance --os-image-api-version 1 image-create \ --name yardstick-trusty-server \ --is-public true --disk-format $DISK_FORMAT \ @@ -297,6 +291,26 @@ load_yardstick_image() echo "Glance image id: $GLANCE_IMAGE_ID" } +report(){ + + echo + echo "========== Reporting Status ==========" + + curl -i -H 'content-type: application/json' -X POST -d \ + "{\"project_name\": \"yardstick\", + \"case_name\": \"scenario_status\", + \"pod_name\":\"${NODE_NAME}\", + \"installer\":\"${INSTALLER_TYPE}\", + \"version\":\"${YARDSTICK_BRANCH}\", + \"scenario\":\"${DEPLOY_SCENARIO}\", + \"description\": \"yardstick ci scenario status\", + \"start_date\":\"$2\", + \"criteria\":\"$1\", + \"stop_date\":\"$3\", + \"details\":\"\"}" \ + ${DISPATCHER_HTTP_TARGET} +} + run_test() { echo @@ -341,7 +355,7 @@ EOF # Mark the test suite failed but continue # running the remaining test suites. - (( failed++ )) + (( ++failed )) fi if [ ${DISPATCHER_TYPE} = file ]; then echo "---------------------------" @@ -363,19 +377,8 @@ EOF if [ $failed -gt 0 ]; then scenario_status="FAILED" fi - curl -i -H 'content-type: application/json' -X POST -d \ - "{\"project_name\": \"yardstick\", - \"case_name\": \"scenario_status\", - \"pod_name\":\"${NODE_NAME}\", - \"installer\":\"${INSTALLER_TYPE}\", - \"version\":\"${YARDSTICK_BRANCH}\", - \"scenario\":\"${DEPLOY_SCENARIO}\", - \"description\": \"yardstick ci scenario status\", - \"start_date\":\"${start_date}\", - \"criteria\":\"${scenario_status}\", - \"stop_date\":\"${stop_date}\", - \"details\":\"\"}" \ - ${DISPATCHER_HTTP_TARGET} + + report $scenario_status $start_date $stop_date if [ $failed -gt 0 ]; then echo "---------------------------" @@ -479,6 +482,12 @@ main() QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-trusty-server.img" + if [ "$DEPLOY_SCENARIO" == "os-nosdn-lxd-ha" -o "$DEPLOY_SCENARIO" == "os-nosdn-lxd-noha" ]; then + DISK_FORMAT="raw" + else + DISK_FORMAT="qcow2" + fi + build_yardstick_image load_yardstick_image if [ $YARD_IMG_ARCH = "arm64" ]; then diff --git a/tools/yardstick-img-modify b/tools/yardstick-img-modify index b9ecbddd8..07f10b3ce 100755 --- a/tools/yardstick-img-modify +++ b/tools/yardstick-img-modify @@ -104,7 +104,7 @@ setup() { kpartx -av $raw_imgfile if [ $YARD_IMG_ARCH = "arm64" ]; then - e2fsck -f /dev/mapper/$loopdevice + e2fsck -p -f /dev/mapper/$loopdevice resize2fs /dev/mapper/$loopdevice fi # for trouble shooting |