summaryrefslogtreecommitdiffstats
path: root/docs/testing/user/userguide
diff options
context:
space:
mode:
Diffstat (limited to 'docs/testing/user/userguide')
-rwxr-xr-xdocs/testing/user/userguide/01-introduction.rst79
-rw-r--r--docs/testing/user/userguide/02-methodology.rst195
-rwxr-xr-xdocs/testing/user/userguide/03-architecture.rst266
-rw-r--r--docs/testing/user/userguide/04-vtc-overview.rst122
-rw-r--r--docs/testing/user/userguide/05-apexlake_installation.rst300
-rw-r--r--docs/testing/user/userguide/06-apexlake_api.rst89
-rw-r--r--docs/testing/user/userguide/07-nsb-overview.rst177
-rw-r--r--docs/testing/user/userguide/08-nsb_installation.rst253
-rw-r--r--docs/testing/user/userguide/09-installation.rst401
-rw-r--r--docs/testing/user/userguide/10-yardstick_plugin.rst144
-rw-r--r--docs/testing/user/userguide/11-result-store-InfluxDB.rst86
-rw-r--r--docs/testing/user/userguide/12-grafana.rst119
-rw-r--r--docs/testing/user/userguide/13-list-of-tcs.rst129
-rwxr-xr-xdocs/testing/user/userguide/Yardstick_task_templates.rst160
-rw-r--r--docs/testing/user/userguide/comp-intro.rst37
-rw-r--r--docs/testing/user/userguide/glossary.rst65
-rwxr-xr-xdocs/testing/user/userguide/images/Deployment.pngbin0 -> 17958 bytes
-rw-r--r--docs/testing/user/userguide/images/Grafana_config.pngbin0 -> 143507 bytes
-rw-r--r--docs/testing/user/userguide/images/InfluxDB_store.pngbin0 -> 1623955 bytes
-rw-r--r--docs/testing/user/userguide/images/Logical_view.pngbin0 -> 58840 bytes
-rw-r--r--docs/testing/user/userguide/images/TC002.pngbin0 -> 106382 bytes
-rw-r--r--docs/testing/user/userguide/images/Use_case.pngbin0 -> 105787 bytes
-rw-r--r--docs/testing/user/userguide/images/add.pngbin0 -> 169904 bytes
-rw-r--r--docs/testing/user/userguide/images/login.pngbin0 -> 32761 bytes
-rw-r--r--docs/testing/user/userguide/images/results_visualization.pngbin0 -> 41905 bytes
-rw-r--r--docs/testing/user/userguide/images/test_execution_flow.pngbin0 -> 51473 bytes
-rw-r--r--docs/testing/user/userguide/index.rst27
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc001.rst133
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc002.rst126
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc004.rst110
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc005.rst125
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc006.rst144
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc007.rst162
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc008.rst90
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc009.rst89
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc010.rst154
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc011.rst123
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc012.rst135
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc014.rst126
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc019.rst134
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc020.rst141
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc021.rst157
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc024.rst76
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc025.rst123
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc027.rst95
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc028.rst70
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc037.rst167
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc038.rst104
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc040.rst65
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc042.rst87
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc043.rst102
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc044.rst82
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc045.rst139
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc046.rst138
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc047.rst139
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc048.rst139
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc049.rst139
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc050.rst135
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc051.rst117
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc052.rst141
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc053.rst142
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc054.rst125
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc055.rst67
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc061.rst88
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc063.rst81
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc069.rst100
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc070.rst110
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc071.rst109
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc072.rst110
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc073.rst81
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc074.rst137
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc075.rst60
-rw-r--r--docs/testing/user/userguide/opnfv_yardstick_tc076.rst61
-rw-r--r--docs/testing/user/userguide/references.rst60
-rw-r--r--docs/testing/user/userguide/testcase_description_v2_template.rst64
75 files changed, 8051 insertions, 0 deletions
diff --git a/docs/testing/user/userguide/01-introduction.rst b/docs/testing/user/userguide/01-introduction.rst
new file mode 100755
index 000000000..0e0eea002
--- /dev/null
+++ b/docs/testing/user/userguide/01-introduction.rst
@@ -0,0 +1,79 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+============
+Introduction
+============
+
+**Welcome to Yardstick's documentation !**
+
+.. _Pharos: https://wiki.opnfv.org/pharos
+.. _Yardstick: https://wiki.opnfv.org/yardstick
+.. _Presentation: https://wiki.opnfv.org/download/attachments/2925202/opnfv_summit_-_yardstick_project.pdf?version=1&modificationDate=1458848320000&api=v2
+Yardstick_ is an OPNFV Project.
+
+The project's goal is to verify infrastructure compliance, from the perspective
+of a Virtual Network Function (:term:`VNF`).
+
+The Project's scope is the development of a test framework, *Yardstick*, test
+cases and test stimuli to enable Network Function Virtualization Infrastructure
+(:term:`NFVI`) verification.
+The Project also includes a sample :term:`VNF`, the Virtual Traffic Classifier
+(:term:`VTC`) and its experimental framework, *ApexLake* !
+
+*Yardstick* is used in OPNFV for verifying the OPNFV infrastructure and some of
+the OPNFV features. The *Yardstick* framework is deployed in several OPNFV
+community labs. It is *installer*, *infrastructure* and *application*
+independent.
+
+.. seealso:: Pharos_ for information on OPNFV community labs and this
+ Presentation_ for an overview of *Yardstick*
+
+
+About This Document
+===================
+
+This document consists of the following chapters:
+
+* Chapter :doc:`02-methodology` describes the methodology implemented by the
+ Yardstick Project for :term:`NFVI` verification.
+
+* Chapter :doc:`03-architecture` provides information on the software architecture
+ of yardstick.
+
+* Chapter :doc:`04-vtc-overview` provides information on the :term:`VTC`.
+
+* Chapter :doc:`05-apexlake_installation` provides instructions to install the
+ experimental framework *ApexLake*
+
+* Chapter :doc:`06-apexlake_api` explains how this framework is integrated in
+ *Yardstick*.
+
+* Chapter :doc:`07-nsb-overview` describes the methodology implemented by the
+ yardstick - Network service benchmarking to test real world usecase for a
+ given VNF
+
+* Chapter :doc:`08-nsb_installation` provides instructions to install
+ *Yardstick - Network service benchmarking testing*.
+
+* Chapter :doc:`09-installation` provides instructions to install *Yardstick*.
+
+* Chapter :doc:`10-yardstick_plugin` provides information on how to integrate
+ other OPNFV testing projects into *Yardstick*.
+
+* Chapter :doc:`11-result-store-InfluxDB` provides inforamtion on how to run
+ plug-in test cases and store test results into community's InfluxDB.
+
+* Chapter :doc:`12-list-of-tcs` includes a list of available Yardstick test
+ cases.
+
+
+Contact Yardstick
+=================
+
+Feedback? `Contact us`_
+
+.. _Contact us: opnfv-users@lists.opnfv.org
+
diff --git a/docs/testing/user/userguide/02-methodology.rst b/docs/testing/user/userguide/02-methodology.rst
new file mode 100644
index 000000000..34d271095
--- /dev/null
+++ b/docs/testing/user/userguide/02-methodology.rst
@@ -0,0 +1,195 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+===========
+Methodology
+===========
+
+Abstract
+========
+
+This chapter describes the methodology implemented by the Yardstick project for
+verifying the :term:`NFVI` from the perspective of a :term:`VNF`.
+
+ETSI-NFV
+========
+
+.. _NFV-TST001: http://www.etsi.org/deliver/etsi_gs/NFV-TST/001_099/001/01.01.01_60/gs_NFV-TST001v010101p.pdf
+.. _Yardsticktst: https://wiki.opnfv.org/download/attachments/2925202/opnfv_summit_-_bridging_opnfv_and_etsi.pdf?version=1&modificationDate=1458848320000&api=v2
+
+
+The document ETSI GS NFV-TST001_, "Pre-deployment Testing; Report on Validation
+of NFV Environments and Services", recommends methods for pre-deployment
+testing of the functional components of an NFV environment.
+
+The Yardstick project implements the methodology described in chapter 6, "Pre-
+deployment validation of NFV infrastructure".
+
+The methodology consists in decomposing the typical :term:`VNF` work-load
+performance metrics into a number of characteristics/performance vectors, which
+each can be represented by distinct test-cases.
+
+The methodology includes five steps:
+
+* *Step1:* Define Infrastruture - the Hardware, Software and corresponding
+ configuration target for validation; the OPNFV infrastructure, in OPNFV
+ community labs.
+
+* *Step2:* Identify :term:`VNF` type - the application for which the
+ infrastructure is to be validated, and its requirements on the underlying
+ infrastructure.
+
+* *Step3:* Select test cases - depending on the workload that represents the
+ application for which the infrastruture is to be validated, the relevant
+ test cases amongst the list of available Yardstick test cases.
+
+* *Step4:* Execute tests - define the duration and number of iterations for the
+ selected test cases, tests runs are automated via OPNFV Jenkins Jobs.
+
+* *Step5:* Collect results - using the common API for result collection.
+
+.. seealso:: Yardsticktst_ for material on alignment ETSI TST001 and Yardstick.
+
+Metrics
+=======
+
+The metrics, as defined by ETSI GS NFV-TST001, are shown in
+:ref:`Table1 <table2_1>`, :ref:`Table2 <table2_2>` and
+:ref:`Table3 <table2_3>`.
+
+In OPNFV Colorado release, generic test cases covering aspects of the listed
+metrics are available; further OPNFV releases will provide extended testing of
+these metrics.
+The view of available Yardstick test cases cross ETSI definitions in
+:ref:`Table1 <table2_1>`, :ref:`Table2 <table2_2>` and :ref:`Table3 <table2_3>`
+is shown in :ref:`Table4 <table2_4>`.
+It shall be noticed that the Yardstick test cases are examples, the test
+duration and number of iterations are configurable, as are the System Under
+Test (SUT) and the attributes (or, in Yardstick nomemclature, the scenario
+options).
+
+.. _table2_1:
+
+**Table 1 - Performance/Speed Metrics**
+
++---------+-------------------------------------------------------------------+
+| Category| Performance/Speed |
+| | |
++---------+-------------------------------------------------------------------+
+| Compute | * Latency for random memory access |
+| | * Latency for cache read/write operations |
+| | * Processing speed (instructions per second) |
+| | * Throughput for random memory access (bytes per second) |
+| | |
++---------+-------------------------------------------------------------------+
+| Network | * Throughput per NFVI node (frames/byte per second) |
+| | * Throughput provided to a VM (frames/byte per second) |
+| | * Latency per traffic flow |
+| | * Latency between VMs |
+| | * Latency between NFVI nodes |
+| | * Packet delay variation (jitter) between VMs |
+| | * Packet delay variation (jitter) between NFVI nodes |
+| | |
++---------+-------------------------------------------------------------------+
+| Storage | * Sequential read/write IOPS |
+| | * Random read/write IOPS |
+| | * Latency for storage read/write operations |
+| | * Throughput for storage read/write operations |
+| | |
++---------+-------------------------------------------------------------------+
+
+.. _table2_2:
+
+**Table 2 - Capacity/Scale Metrics**
+
++---------+-------------------------------------------------------------------+
+| Category| Capacity/Scale |
+| | |
++---------+-------------------------------------------------------------------+
+| Compute | * Number of cores and threads- Available memory size |
+| | * Cache size |
+| | * Processor utilization (max, average, standard deviation) |
+| | * Memory utilization (max, average, standard deviation) |
+| | * Cache utilization (max, average, standard deviation) |
+| | |
++---------+-------------------------------------------------------------------+
+| Network | * Number of connections |
+| | * Number of frames sent/received |
+| | * Maximum throughput between VMs (frames/byte per second) |
+| | * Maximum throughput between NFVI nodes (frames/byte per second) |
+| | * Network utilization (max, average, standard deviation) |
+| | * Number of traffic flows |
+| | |
++---------+-------------------------------------------------------------------+
+| Storage | * Storage/Disk size |
+| | * Capacity allocation (block-based, object-based) |
+| | * Block size |
+| | * Maximum sequential read/write IOPS |
+| | * Maximum random read/write IOPS |
+| | * Disk utilization (max, average, standard deviation) |
+| | |
++---------+-------------------------------------------------------------------+
+
+.. _table2_3:
+
+**Table 3 - Availability/Reliability Metrics**
+
++---------+-------------------------------------------------------------------+
+| Category| Availability/Reliability |
+| | |
++---------+-------------------------------------------------------------------+
+| Compute | * Processor availability (Error free processing time) |
+| | * Memory availability (Error free memory time) |
+| | * Processor mean-time-to-failure |
+| | * Memory mean-time-to-failure |
+| | * Number of processing faults per second |
+| | |
++---------+-------------------------------------------------------------------+
+| Network | * NIC availability (Error free connection time) |
+| | * Link availability (Error free transmission time) |
+| | * NIC mean-time-to-failure |
+| | * Network timeout duration due to link failure |
+| | * Frame loss rate |
+| | |
++---------+-------------------------------------------------------------------+
+| Storage | * Disk availability (Error free disk access time) |
+| | * Disk mean-time-to-failure |
+| | * Number of failed storage read/write operations per second |
+| | |
++---------+-------------------------------------------------------------------+
+
+.. _table2_4:
+
+**Table 4 - Yardstick Generic Test Cases**
+
++---------+-------------------+----------------+------------------------------+
+| Category| Performance/Speed | Capacity/Scale | Availability/Reliability |
+| | | | |
++---------+-------------------+----------------+------------------------------+
+| Compute | TC003 [1]_ | TC003 [1]_ | TC013 [1]_ |
+| | TC004 | TC004 | TC015 [1]_ |
+| | TC010 | TC024 | |
+| | TC012 | TC055 | |
+| | TC014 | | |
+| | TC069 | | |
++---------+-------------------+----------------+------------------------------+
+| Network | TC001 | TC044 | TC016 [1]_ |
+| | TC002 | TC073 | TC018 [1]_ |
+| | TC009 | TC075 | |
+| | TC011 | | |
+| | TC042 | | |
+| | TC043 | | |
++---------+-------------------+----------------+------------------------------+
+| Storage | TC005 | TC063 | TC017 [1]_ |
++---------+-------------------+----------------+------------------------------+
+
+.. note:: The description in this OPNFV document is intended as a reference for
+ users to understand the scope of the Yardstick Project and the
+ deliverables of the Yardstick framework. For complete description of
+ the methodology, please refer to the ETSI document.
+
+.. rubric:: Footnotes
+.. [1] To be included in future deliveries.
+
diff --git a/docs/testing/user/userguide/03-architecture.rst b/docs/testing/user/userguide/03-architecture.rst
new file mode 100755
index 000000000..03bf00f58
--- /dev/null
+++ b/docs/testing/user/userguide/03-architecture.rst
@@ -0,0 +1,266 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) 2016 Huawei Technologies Co.,Ltd and others
+
+============
+Architecture
+============
+
+Abstract
+========
+This chapter describes the yardstick framework software architecture. we will introduce it from Use-Case View,
+Logical View, Process View and Deployment View. More technical details will be introduced in this chapter.
+
+Overview
+========
+
+Architecture overview
+---------------------
+Yardstick is mainly written in Python, and test configurations are made
+in YAML. Documentation is written in reStructuredText format, i.e. .rst
+files. Yardstick is inspired by Rally. Yardstick is intended to run on a
+computer with access and credentials to a cloud. The test case is described
+in a configuration file given as an argument.
+
+How it works: the benchmark task configuration file is parsed and converted into
+an internal model. The context part of the model is converted into a Heat
+template and deployed into a stack. Each scenario is run using a runner, either
+serially or in parallel. Each runner runs in its own subprocess executing
+commands in a VM using SSH. The output of each scenario is written as json
+records to a file or influxdb or http server, we use influxdb as the backend,
+the test result will be shown with grafana.
+
+
+Concept
+-------
+**Benchmark** - assess the relative performance of something
+
+**Benchmark** configuration file - describes a single test case in yaml format
+
+**Context** - The set of Cloud resources used by a scenario, such as user
+names, image names, affinity rules and network configurations. A context is
+converted into a simplified Heat template, which is used to deploy onto the
+Openstack environment.
+
+**Data** - Output produced by running a benchmark, written to a file in json format
+
+**Runner** - Logic that determines how a test scenario is run and reported, for
+example the number of test iterations, input value stepping and test duration.
+Predefined runner types exist for re-usage, see `Runner types`_.
+
+**Scenario** - Type/class of measurement for example Ping, Pktgen, (Iperf, LmBench, ...)
+
+**SLA** - Relates to what result boundary a test case must meet to pass. For
+example a latency limit, amount or ratio of lost packets and so on. Action
+based on :term:`SLA` can be configured, either just to log (monitor) or to stop
+further testing (assert). The :term:`SLA` criteria is set in the benchmark
+configuration file and evaluated by the runner.
+
+
+Runner types
+------------
+
+There exists several predefined runner types to choose between when designing
+a test scenario:
+
+**Arithmetic:**
+Every test run arithmetically steps the specified input value(s) in the
+test scenario, adding a value to the previous input value. It is also possible
+to combine several input values for the same test case in different
+combinations.
+
+Snippet of an Arithmetic runner configuration:
+::
+
+
+ runner:
+ type: Arithmetic
+ iterators:
+ -
+ name: stride
+ start: 64
+ stop: 128
+ step: 64
+
+**Duration:**
+The test runs for a specific period of time before completed.
+
+Snippet of a Duration runner configuration:
+::
+
+
+ runner:
+ type: Duration
+ duration: 30
+
+**Sequence:**
+The test changes a specified input value to the scenario. The input values
+to the sequence are specified in a list in the benchmark configuration file.
+
+Snippet of a Sequence runner configuration:
+::
+
+
+ runner:
+ type: Sequence
+ scenario_option_name: packetsize
+ sequence:
+ - 100
+ - 200
+ - 250
+
+
+**Iteration:**
+Tests are run a specified number of times before completed.
+
+Snippet of an Iteration runner configuration:
+::
+
+
+ runner:
+ type: Iteration
+ iterations: 2
+
+
+
+
+Use-Case View
+=============
+Yardstick Use-Case View shows two kinds of users. One is the Tester who will
+do testing in cloud, the other is the User who is more concerned with test result
+and result analyses.
+
+For testers, they will run a single test case or test case suite to verify
+infrastructure compliance or bencnmark their own infrastructure performance.
+Test result will be stored by dispatcher module, three kinds of store method
+(file, influxdb and http) can be configured. The detail information of
+scenarios and runners can be queried with CLI by testers.
+
+For users, they would check test result with four ways.
+
+If dispatcher module is configured as file(default), there are two ways to
+check test result. One is to get result from yardstick.out ( default path:
+/tmp/yardstick.out), the other is to get plot of test result, it will be shown
+if users execute command "yardstick-plot".
+
+If dispatcher module is configured as influxdb, users will check test
+result on Grafana which is most commonly used for visualizing time series data.
+
+If dispatcher module is configured as http, users will check test result
+on OPNFV testing dashboard which use MongoDB as backend.
+
+.. image:: images/Use_case.png
+ :width: 800px
+ :alt: Yardstick Use-Case View
+
+Logical View
+============
+Yardstick Logical View describes the most important classes, their
+organization, and the most important use-case realizations.
+
+Main classes:
+
+**TaskCommands** - "yardstick task" subcommand handler.
+
+**HeatContext** - Do test yaml file context section model convert to HOT,
+deploy and undeploy Openstack heat stack.
+
+**Runner** - Logic that determines how a test scenario is run and reported.
+
+**TestScenario** - Type/class of measurement for example Ping, Pktgen, (Iperf,
+LmBench, ...)
+
+**Dispatcher** - Choose user defined way to store test results.
+
+TaskCommands is the "yardstick task" subcommand's main entry. It takes yaml
+file (e.g. test.yaml) as input, and uses HeatContext to convert the yaml
+file's context section to HOT. After Openstack heat stack is deployed by
+HeatContext with the converted HOT, TaskCommands use Runner to run specified
+TestScenario. During first runner initialization, it will create output
+process. The output process use Dispatcher to push test results. The Runner
+will also create a process to execute TestScenario. And there is a
+multiprocessing queue between each runner process and output process, so the
+runner process can push the real-time test results to the storage media.
+TestScenario is commonly connected with VMs by using ssh. It sets up VMs and
+run test measurement scripts through the ssh tunnel. After all TestScenaio
+is finished, TaskCommands will undeploy the heat stack. Then the whole test is
+finished.
+
+.. image:: images/Logical_view.png
+ :width: 800px
+ :alt: Yardstick Logical View
+
+Process View (Test execution flow)
+==================================
+Yardstick process view shows how yardstick runs a test case. Below is the
+sequence graph about the test execution flow using heat context, and each
+object represents one module in yardstick:
+
+.. image:: images/test_execution_flow.png
+ :width: 800px
+ :alt: Yardstick Process View
+
+A user wants to do a test with yardstick. He can use the CLI to input the
+command to start a task. "TaskCommands" will receive the command and ask
+"HeatContext" to parse the context. "HeatContext" will then ask "Model" to
+convert the model. After the model is generated, "HeatContext" will inform
+"Openstack" to deploy the heat stack by heat template. After "Openstack"
+deploys the stack, "HeatContext" will inform "Runner" to run the specific test
+case.
+
+Firstly, "Runner" would ask "TestScenario" to process the specific scenario.
+Then "TestScenario" will start to log on the openstack by ssh protocal and
+execute the test case on the specified VMs. After the script execution
+finishes, "TestScenario" will send a message to inform "Runner". When the
+testing job is done, "Runner" will inform "Dispatcher" to output the test
+result via file, influxdb or http. After the result is output, "HeatContext"
+will call "Openstack" to undeploy the heat stack. Once the stack is
+undepoyed, the whole test ends.
+
+Deployment View
+===============
+Yardstick deployment view shows how the yardstick tool can be deployed into the
+underlying platform. Generally, yardstick tool is installed on JumpServer(see
+`07-installation` for detail installation steps), and JumpServer is
+connected with other control/compute servers by networking. Based on this
+deployment, yardstick can run the test cases on these hosts, and get the test
+result for better showing.
+
+.. image:: images/Deployment.png
+ :width: 800px
+ :alt: Yardstick Deployment View
+
+Yardstick Directory structure
+=============================
+
+**yardstick/** - Yardstick main directory.
+
+*ci/* - Used for continuous integration of Yardstick at different PODs and
+ with support for different installers.
+
+*docs/* - All documentation is stored here, such as configuration guides,
+ user guides and Yardstick descriptions.
+
+*etc/* - Used for test cases requiring specific POD configurations.
+
+*samples/* - test case samples are stored here, most of all scenario and
+ feature's samples are shown in this directory.
+
+*tests/* - Here both Yardstick internal tests (*functional/* and *unit/*) as
+ well as the test cases run to verify the NFVI (*opnfv/*) are stored.
+ Also configurations of what to run daily and weekly at the different
+ PODs is located here.
+
+*tools/* - Currently contains tools to build image for VMs which are deployed
+ by Heat. Currently contains how to build the yardstick-trusty-server
+ image with the different tools that are needed from within the image.
+
+*plugin/* - Plug-in configuration files are stored here.
+
+*vTC/* - Contains the files for running the virtual Traffic Classifier tests.
+
+*yardstick/* - Contains the internals of Yardstick: Runners, Scenario, Contexts,
+ CLI parsing, keys, plotting tools, dispatcher, plugin
+ install/remove scripts and so on.
+
diff --git a/docs/testing/user/userguide/04-vtc-overview.rst b/docs/testing/user/userguide/04-vtc-overview.rst
new file mode 100644
index 000000000..82b20cad5
--- /dev/null
+++ b/docs/testing/user/userguide/04-vtc-overview.rst
@@ -0,0 +1,122 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, National Center of Scientific Research "Demokritos" and others.
+
+==========================
+Virtual Traffic Classifier
+==========================
+
+Abstract
+========
+
+.. _TNOVA: http://www.t-nova.eu/
+.. _TNOVAresults: http://www.t-nova.eu/results/
+.. _Yardstick: https://wiki.opnfv.org/yardstick
+
+This chapter provides an overview of the virtual Traffic Classifier, a
+contribution to OPNFV Yardstick_ from the EU Project TNOVA_.
+Additional documentation is available in TNOVAresults_.
+
+Overview
+========
+
+The virtual Traffic Classifier (:term:`VTC`) :term:`VNF`, comprises of a
+Virtual Network Function Component (:term:`VNFC`). The :term:`VNFC` contains
+both the Traffic Inspection module, and the Traffic forwarding module, needed
+to run the :term:`VNF`. The exploitation of Deep Packet Inspection
+(:term:`DPI`) methods for traffic classification is built around two basic
+assumptions:
+
+* third parties unaffiliated with either source or recipient are able to
+inspect each IP packet’s payload
+
+* the classifier knows the relevant syntax of each application’s packet
+payloads (protocol signatures, data patterns, etc.).
+
+The proposed :term:`DPI` based approach will only use an indicative, small
+number of the initial packets from each flow in order to identify the content
+and not inspect each packet.
+
+In this respect it follows the Packet Based per Flow State (term:`PBFS`). This
+method uses a table to track each session based on the 5-tuples (src address,
+dest address, src port,dest port, transport protocol) that is maintained for
+each flow.
+
+Concepts
+========
+
+* *Traffic Inspection*: The process of packet analysis and application
+identification of network traffic that passes through the :term:`VTC`.
+
+* *Traffic Forwarding*: The process of packet forwarding from an incoming
+network interface to a pre-defined outgoing network interface.
+
+* *Traffic Rule Application*: The process of packet tagging, based on a
+predefined set of rules. Packet tagging may include e.g. Type of Service
+(:term:`ToS`) field modification.
+
+Architecture
+============
+
+The Traffic Inspection module is the most computationally intensive component
+of the :term:`VNF`. It implements filtering and packet matching algorithms in
+order to support the enhanced traffic forwarding capability of the :term:`VNF`.
+The component supports a flow table (exploiting hashing algorithms for fast
+indexing of flows) and an inspection engine for traffic classification.
+
+The implementation used for these experiments exploits the nDPI library.
+The packet capturing mechanism is implemented using libpcap. When the
+:term:`DPI` engine identifies a new flow, the flow register is updated with the
+appropriate information and transmitted across the Traffic Forwarding module,
+which then applies any required policy updates.
+
+The Traffic Forwarding moudle is responsible for routing and packet forwarding.
+It accepts incoming network traffic, consults the flow table for classification
+information for each incoming flow and then applies pre-defined policies
+marking e.g. :term:`ToS`/Differentiated Services Code Point (:term:`DSCP`)
+multimedia traffic for Quality of Service (:term:`QoS`) enablement on the
+forwarded traffic.
+It is assumed that the traffic is forwarded using the default policy until it
+is identified and new policies are enforced.
+
+The expected response delay is considered to be negligible, as only a small
+number of packets are required to identify each flow.
+
+Graphical Overview
+==================
+
+.. code-block:: console
+
+ +----------------------------+
+ | |
+ | Virtual Traffic Classifier |
+ | |
+ | Analysing/Forwarding |
+ | ------------> |
+ | ethA ethB |
+ | |
+ +----------------------------+
+ | ^
+ | |
+ v |
+ +----------------------------+
+ | |
+ | Virtual Switch |
+ | |
+ +----------------------------+
+
+Install
+=======
+
+run the build.sh with root privileges
+
+Run
+===
+
+sudo ./pfbridge -a eth1 -b eth2
+
+Development Environment
+=======================
+
+Ubuntu 14.04
diff --git a/docs/testing/user/userguide/05-apexlake_installation.rst b/docs/testing/user/userguide/05-apexlake_installation.rst
new file mode 100644
index 000000000..d4493e0f8
--- /dev/null
+++ b/docs/testing/user/userguide/05-apexlake_installation.rst
@@ -0,0 +1,300 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Intel Corporation and others.
+
+
+.. _DPDK: http://dpdk.org/doc/nics
+.. _DPDK-pktgen: https://github.com/Pktgen/Pktgen-DPDK/
+.. _SRIOV: https://wiki.openstack.org/wiki/SR-IOV-Passthrough-For-Networking
+.. _PORTSEC: https://wiki.openstack.org/wiki/Neutron/ML2PortSecurityExtensionDriver
+.. _here: https://wiki.opnfv.org/vtc
+
+
+============================
+Apexlake Installation Guide
+============================
+
+Abstract
+--------
+
+ApexLake is a framework that provides automatic execution of experiments and
+related data collection to enable a user validate infrastructure from the
+perspective of a Virtual Network Function (:term:`VNF`).
+
+In the context of Yardstick, a virtual Traffic Classifier (:term:`VTC`) network
+function is utilized.
+
+
+Framework Hardware Dependencies
+===============================
+
+In order to run the framework there are some hardware related dependencies for
+ApexLake.
+
+The framework needs to be installed on the same physical node where DPDK-pktgen_
+is installed.
+
+The installation requires the physical node hosting the packet generator must
+have 2 NICs which are DPDK_ compatible.
+
+The 2 NICs will be connected to the switch where the OpenStack VM
+network is managed.
+
+The switch used must support multicast traffic and :term:`IGMP` snooping.
+Further details about the configuration are provided at the following here_.
+
+The corresponding ports to which the cables are connected need to be configured
+as VLAN trunks using two of the VLAN IDs available for Neutron.
+Note the VLAN IDs used as they will be required in later configuration steps.
+
+
+Framework Software Dependencies
+===============================
+Before starting the framework, a number of dependencies must first be installed.
+The following describes the set of instructions to be executed via the Linux
+shell in order to install and configure the required dependencies.
+
+1. Install Dependencies.
+
+To support the framework dependencies the following packages must be installed.
+The example provided is based on Ubuntu and needs to be executed in root mode.
+
+::
+
+ apt-get install python-dev
+ apt-get install python-pip
+ apt-get install python-mock
+ apt-get install tcpreplay
+ apt-get install libpcap-dev
+
+2. Source OpenStack openrc file.
+
+::
+
+ source openrc
+
+3. Configure Openstack Neutron
+
+In order to support traffic generation and management by the virtual
+Traffic Classifier, the configuration of the port security driver
+extension is required for Neutron.
+
+For further details please follow the following link: PORTSEC_
+This step can be skipped in case the target OpenStack is Juno or Kilo release,
+but it is required to support Liberty.
+It is therefore required to indicate the release version in the configuration
+file located in ./yardstick/vTC/apexlake/apexlake.conf
+
+
+4. Create Two Networks based on VLANs in Neutron.
+
+To enable network communications between the packet generator and the compute
+node, two networks must be created via Neutron and mapped to the VLAN IDs
+that were previously used in the configuration of the physical switch.
+The following shows the typical set of commands required to configure Neutron
+correctly.
+The physical switches need to be configured accordingly.
+
+::
+
+ VLAN_1=2032
+ VLAN_2=2033
+ PHYSNET=physnet2
+ neutron net-create apexlake_inbound_network \
+ --provider:network_type vlan \
+ --provider:segmentation_id $VLAN_1 \
+ --provider:physical_network $PHYSNET
+
+ neutron subnet-create apexlake_inbound_network \
+ 192.168.0.0/24 --name apexlake_inbound_subnet
+
+ neutron net-create apexlake_outbound_network \
+ --provider:network_type vlan \
+ --provider:segmentation_id $VLAN_2 \
+ --provider:physical_network $PHYSNET
+
+ neutron subnet-create apexlake_outbound_network 192.168.1.0/24 \
+ --name apexlake_outbound_subnet
+
+
+5. Download Ubuntu Cloud Image and load it on Glance
+
+The virtual Traffic Classifier is supported on top of Ubuntu 14.04 cloud image.
+The image can be downloaded on the local machine and loaded on Glance
+using the following commands:
+
+::
+
+ wget cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
+ glance image-create \
+ --name ubuntu1404 \
+ --is-public true \
+ --disk-format qcow \
+ --container-format bare \
+ --file trusty-server-cloudimg-amd64-disk1.img
+
+
+
+6. Configure the Test Cases
+
+The VLAN tags must also be included in the test case Yardstick yaml file
+as parameters for the following test cases:
+
+ * :doc:`opnfv_yardstick_tc006`
+
+ * :doc:`opnfv_yardstick_tc007`
+
+ * :doc:`opnfv_yardstick_tc020`
+
+ * :doc:`opnfv_yardstick_tc021`
+
+
+Install and Configure DPDK Pktgen
++++++++++++++++++++++++++++++++++
+
+Execution of the framework is based on DPDK Pktgen.
+If DPDK Pktgen has not installed, it is necessary to download, install, compile
+and configure it.
+The user can create a directory and download the dpdk packet generator source
+code:
+
+::
+
+ cd experimental_framework/libraries
+ mkdir dpdk_pktgen
+ git clone https://github.com/pktgen/Pktgen-DPDK.git
+
+For instructions on the installation and configuration of DPDK and DPDK Pktgen
+please follow the official DPDK Pktgen README file.
+Once the installation is completed, it is necessary to load the DPDK kernel
+driver, as follow:
+
+::
+
+ insmod uio
+ insmod DPDK_DIR/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
+
+It is necessary to set the configuration file to support the desired Pktgen
+configuration.
+A description of the required configuration parameters and supporting examples
+is provided in the following:
+
+::
+
+ [PacketGen]
+ packet_generator = dpdk_pktgen
+
+ # This is the directory where the packet generator is installed
+ # (if the user previously installed dpdk-pktgen,
+ # it is required to provide the director where it is installed).
+ pktgen_directory = /home/user/software/dpdk_pktgen/dpdk/examples/pktgen/
+
+ # This is the directory where DPDK is installed
+ dpdk_directory = /home/user/apexlake/experimental_framework/libraries/Pktgen-DPDK/dpdk/
+
+ # Name of the dpdk-pktgen program that starts the packet generator
+ program_name = app/app/x86_64-native-linuxapp-gcc/pktgen
+
+ # DPDK coremask (see DPDK-Pktgen readme)
+ coremask = 1f
+
+ # DPDK memory channels (see DPDK-Pktgen readme)
+ memory_channels = 3
+
+ # Name of the interface of the pktgen to be used to send traffic (vlan_sender)
+ name_if_1 = p1p1
+
+ # Name of the interface of the pktgen to be used to receive traffic (vlan_receiver)
+ name_if_2 = p1p2
+
+ # PCI bus address correspondent to if_1
+ bus_slot_nic_1 = 01:00.0
+
+ # PCI bus address correspondent to if_2
+ bus_slot_nic_2 = 01:00.1
+
+
+To find the parameters related to names of the NICs and the addresses of the PCI buses
+the user may find it useful to run the :term:`DPDK` tool nic_bind as follows:
+
+::
+
+ DPDK_DIR/tools/dpdk_nic_bind.py --status
+
+Lists the NICs available on the system, and shows the available drivers and bus addresses for each interface.
+Please make sure to select NICs which are :term:`DPDK` compatible.
+
+Installation and Configuration of smcroute
+++++++++++++++++++++++++++++++++++++++++++
+
+The user is required to install smcroute which is used by the framework to
+support multicast communications.
+
+The following is the list of commands required to download and install smroute.
+
+::
+
+ cd ~
+ git clone https://github.com/troglobit/smcroute.git
+ cd smcroute
+ git reset --hard c3f5c56
+ sed -i 's/aclocal-1.11/aclocal/g' ./autogen.sh
+ sed -i 's/automake-1.11/automake/g' ./autogen.sh
+ ./autogen.sh
+ ./configure
+ make
+ sudo make install
+ cd ..
+
+It is required to do the reset to the specified commit ID.
+It is also requires the creation a configuration file using the following
+command:
+
+ SMCROUTE_NIC=(name of the nic)
+
+where name of the nic is the name used previously for the variable "name_if_2".
+For example:
+
+::
+
+ SMCROUTE_NIC=p1p2
+
+Then create the smcroute configuration file /etc/smcroute.conf
+
+::
+
+ echo mgroup from $SMCROUTE_NIC group 224.192.16.1 > /etc/smcroute.conf
+
+
+At the end of this procedure it will be necessary to perform the following
+actions to add the user to the sudoers:
+
+::
+
+ adduser USERNAME sudo
+ echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+
+Experiment using SR-IOV Configuration on the Compute Node
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+To enable :term:`SR-IOV` interfaces on the physical NIC of the compute node, a
+compatible NIC is required.
+NIC configuration depends on model and vendor. After proper configuration to
+support :term:`SR-IOV`, a proper configuration of OpenStack is required.
+For further information, please refer to the SRIOV_ configuration guide
+
+Finalize installation the framework on the system
+=================================================
+
+The installation of the framework on the system requires the setup of the project.
+After entering into the apexlake directory, it is sufficient to run the following
+command.
+
+::
+
+ python setup.py install
+
+Since some elements are copied into the /tmp directory (see configuration file)
+it could be necessary to repeat this step after a reboot of the host.
diff --git a/docs/testing/user/userguide/06-apexlake_api.rst b/docs/testing/user/userguide/06-apexlake_api.rst
new file mode 100644
index 000000000..35a1dbe3e
--- /dev/null
+++ b/docs/testing/user/userguide/06-apexlake_api.rst
@@ -0,0 +1,89 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Intel Corporation and others.
+
+
+=================================
+Apexlake API Interface Definition
+=================================
+
+Abstract
+--------
+
+The API interface provided by the framework to enable the execution of test
+cases is defined as follows.
+
+
+init
+----
+
+**static init()**
+
+ Initializes the Framework
+
+ **Returns** None
+
+
+execute_framework
+-----------------
+
+**static execute_framework** (test_cases,
+
+ iterations,
+
+ heat_template,
+
+ heat_template_parameters,
+
+ deployment_configuration,
+
+ openstack_credentials)
+
+ Executes the framework according the specified inputs
+
+ **Parameters**
+
+ - **test_cases**
+
+ Test cases to be run with the workload (dict() of dict())
+
+ Example:
+ test_case = dict()
+
+ test_case[’name’] = ‘module.Class’
+
+ test_case[’params’] = dict()
+
+ test_case[’params’][’throughput’] = ‘1’
+
+ test_case[’params’][’vlan_sender’] = ‘1000’
+
+ test_case[’params’][’vlan_receiver’] = ‘1001’
+
+ test_cases = [test_case]
+
+ - **iterations**
+ Number of test cycles to be executed (int)
+
+ - **heat_template**
+ (string) File name of the heat template corresponding to the workload to be deployed.
+ It contains the parameters to be evaluated in the form of #parameter_name.
+ (See heat_templates/vTC.yaml as example).
+
+ - **heat_template_parameters**
+ (dict) Parameters to be provided as input to the
+ heat template. See http://docs.openstack.org/developer/heat/ template_guide/hot_guide.html
+ section “Template input parameters” for further info.
+
+ - **deployment_configuration**
+ ( dict[string] = list(strings) ) ) Dictionary of parameters
+ representing the deployment configuration of the workload.
+
+ The key is a string corresponding to the name of the parameter,
+ the value is a list of strings representing the value to be
+ assumed by a specific param. The parameters are user defined:
+ they have to correspond to the place holders (#parameter_name)
+ specified in the heat template.
+
+ **Returns** dict() containing results
diff --git a/docs/testing/user/userguide/07-nsb-overview.rst b/docs/testing/user/userguide/07-nsb-overview.rst
new file mode 100644
index 000000000..19719f1a7
--- /dev/null
+++ b/docs/testing/user/userguide/07-nsb-overview.rst
@@ -0,0 +1,177 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, 2016-2017 Intel Corporation.
+
+=====================================
+Network Services Benchmarking (NSB)
+=====================================
+
+Abstract
+========
+
+.. _Yardstick: https://wiki.opnfv.org/yardstick
+
+This chapter provides an overview of the NSB, a contribution to OPNFV
+Yardstick_ from Intel.
+
+Overview
+========
+
+GOAL: Extend Yardstick to perform real world VNFs and NFVi Characterization and
+benchmarking with repeatable and deterministic methods.
+
+The Network Service Benchmarking (NSB) extends the yardstick framework to do
+VNF characterization and benchmarking in three different execution
+environments viz., bare metal i.e. native Linux environment, standalone virtual
+environment and managed virtualized environment (e.g. Open stack etc.).
+It also brings in the capability to interact with external traffic generators
+both hardware & software based for triggering and validating the traffic
+according to user defined profiles.
+
+NSB extension includes:
+ • Generic data models of Network Services, based on ETSI specs
+ • New Standalone context for VNF testing like SRIOV, OVS, OVS-DPDK etc
+ • Generic VNF configuration models and metrics implemented with Python
+ classes
+ • Traffic generator features and traffic profiles
+ • L1-L3 state-less traffic profiles
+ • L4-L7 state-full traffic profiles
+ • Tunneling protocol / network overlay support
+ • Test case samples
+ • Ping
+ • Trex
+ • vPE,vCGNAT, vFirewall etc - ipv4 throughput, latency etc
+ • Traffic generators like Trex, ab/nginx, ixia, iperf etc
+ • KPIs for a given use case:
+ • System agent support for collecting NFvi KPI. This includes:
+ o CPU statistic
+ o Memory BW
+ o OVS-DPDK Stats
+ • Network KPIs – eg, inpackets, outpackets, thoughput, latency etc
+ • VNF KPIs – packet_in, packet_drop, packet_fwd etc
+
+Architecture
+============
+The Network Service (NS) defines a set of Virtual Network Functions (VNF)
+connected together using NFV infrastructure.
+
+The Yardstick NSB extension can support multiple VNFs created by different
+vendors including traffic generators. Every VNF being tested has its
+own data model. The Network service defines a VNF modelling on base of performed
+network functionality. The part of the data model is a set of the configuration
+parameters, number of connection points used and flavor including core and
+memory amount.
+
+The ETSI defines a Network Service as a set of configurable VNFs working in
+some NFV Infrastructure connecting each other using Virtual Links available
+through Connection Points. The ETSI MANO specification defines a set of
+management entities called Network Service Descriptors (NSD) and
+VNF Descriptors (VNFD) that define real Network Service. The picture below
+makes an example how the real Network Operator use-case can map into ETSI
+Network service definition
+
+Network Service framework performs the necessary test steps. It may involve
+ o Interacting with traffic generator and providing the inputs on traffic
+ type / packet structure to generate the required traffic as per the
+ test case. Traffic profiles will be used for this.
+ o Executing the commands required for the test procedure and analyses the
+ command output for confirming whether the command got executed correctly
+ or not. E.g. As per the test case, run the traffic for the given
+ time period / wait for the necessary time delay
+ o Verify the test result.
+ o Validate the traffic flow from SUT
+ o Fetch the table / data from SUT and verify the value as per the test case
+ o Upload the logs from SUT onto the Test Harness server
+ o Read the KPI’s provided by particular VNF
+
+Components of Network Service
+------------------------------
+
+* *Models for Network Service benchmarking*: The Network Service benchmarking
+ requires the proper modelling approach. The NSB provides models using Python
+ files and defining of NSDs and VNFDs.
+
+The benchmark control application being a part of OPNFV yardstick can call
+that python models to instantiate and configure the VNFs. Depending on
+infrastructure type (bare-metal or fully virtualized) that calls could be
+made directly or using MANO system.
+
+* *Traffic generators in NSB*: Any benchmark application requires a set of
+ traffic generator and traffic profiles defining the method in which traffic
+ is generated.
+
+The Network Service benchmarking model extends the Network Service
+definition with a set of Traffic Generators (TG) that are treated
+same way as other VNFs being a part of benchmarked network service.
+Same as other VNFs the traffic generator are instantiated and terminated.
+
+Every traffic generator has own configuration defined as a traffic profile and
+a set of KPIs supported. The python models for TG is extended by specific calls
+to listen and generate traffic.
+
+* *The stateless TREX traffic generator*: The main traffic generator used as
+ Network Service stimulus is open source TREX tool.
+
+The TREX tool can generate any kind of stateless traffic.
+
+.. code-block:: console
+
+ +--------+ +-------+ +--------+
+ | | | | | |
+ | Trex | ---> | VNF | ---> | Trex |
+ | | | | | |
+ +--------+ +-------+ +--------+
+
+Supported testcases scenarios:
+• Correlated UDP traffic using TREX traffic generator and replay VNF.
+ o using different IMIX configuration like pure voice, pure video traffic etc
+ o using different number IP flows like 1 flow, 1K, 16K, 64K, 256K, 1M flows
+ o Using different number of rules configured like 1 rule, 1K, 10K rules
+
+For UDP correlated traffic following Key Performance Indicators are collected
+for every combination of test case parameters:
+ • RFC2544 throughput for various loss rate defined (1% is a default)
+
+Graphical Overview
+==================
+
+NSB Testing with yardstick framework facilitate performance testing of various
+VNFs provided.
+
+.. code-block:: console
+ +-----------+
+ | | +-----------+
+ | vPE | ->|TGen Port 0|
+ | TestCase | | +-----------+
+ | | |
+ +-----------+ +------------------+ +-------+ |
+ | | -- API --> | VNF | <--->
+ +-----------+ | Yardstick | +-------+ |
+ | Test Case | --> | NSB Testing | |
+ +-----------+ | | |
+ | | | |
+ | +------------------+ |
+ +-----------+ | +-----------+
+ | Traffic | ->|TGen Port 1|
+ | patterns | +-----------+
+ +-----------+
+ Figure 1: Network Service - 2 server configuration
+
+
+Install
+=======
+
+run the nsb_install.sh with root privileges
+
+Run
+===
+
+source ~/.bash_profile
+cd <yardstick_repo>/yardstick/cmd
+sudo -E ./NSBperf.py --vnf vpe --test tc_baremetal_rfc2544_ipv4_1flow_64B.yaml
+
+Development Environment
+=======================
+
+Ubuntu 14.04, Ubuntu 16.04
diff --git a/docs/testing/user/userguide/08-nsb_installation.rst b/docs/testing/user/userguide/08-nsb_installation.rst
new file mode 100644
index 000000000..a390bb7d7
--- /dev/null
+++ b/docs/testing/user/userguide/08-nsb_installation.rst
@@ -0,0 +1,253 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, 2016-2017 Intel Corporation.
+
+Yardstick - NSB Testing -Installation
+=====================================
+
+Abstract
+--------
+
+Yardstick supports installation on Ubuntu 14.04 or via a Docker image. The
+installation procedure on Ubuntu 14.04 or via the docker image are detailed in
+the section below.
+
+The Network Service Benchmarking (NSB) extends the yardstick framework to do
+VNF characterization and benchmarking in three different execution
+environments viz., bare metal i.e. native Linux environment, standalone virtual
+environment and managed virtualized environment (e.g. Open stack etc.).
+It also brings in the capability to interact with external traffic generators
+both hardware & software based for triggering and validating the traffic
+according to user defined profiles.
+
+The steps needed to run Yardstick with NSB testing are:
+
+* Install Yardstick (NSB Testing).
+* Setup pod.yaml describing Test topology
+* Create the test configuration yaml file.
+* Run the test case.
+
+
+Prerequisites
+-------------
+
+Refer chapter 08-instalaltion.rst for more information on yardstick
+prerequisites
+
+Several prerequisites are needed for Yardstick(VNF testing):
+* Python Modules: pyzmq, pika.
+* flex
+* bison
+* build-essential
+* automake
+* libtool
+* librabbitmq-dev
+* rabbitmq-server
+* collectd
+* intel-cmt-cat
+
+Installing Yardstick on Ubuntu 14.04
+------------------------------------
+
+.. _install-framework:
+
+You can install Yardstick framework directly on Ubuntu 14.04 or in an Ubuntu
+14.04 Docker image. No matter which way you choose to install Yardstick
+framework, the following installation steps are identical.
+
+If you choose to use the Ubuntu 14.04 Docker image, You can pull the Ubuntu
+14.04 Docker image from Docker hub:
+
+::
+
+ docker pull ubuntu:14.04
+
+Installing Yardstick framework
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Download source code and install python dependencies:
+
+::
+
+ git clone https://gerrit.opnfv.org/gerrit/yardstick
+ cd yardstick
+ ./nsb_setup.sh
+
+It will automatically download all the packages needed for NSB Testing setup.
+
+System Topology:
+-----------------
+
+.. code-block:: console
+
+ +----------+ +----------+
+ | | | |
+ | | (0)----->(0) | Ping/ |
+ | TG1 | | vPE/ |
+ | | | 2Trex |
+ | | (1)<-----(1) | |
+ +----------+ +----------+
+ trafficgen_1 vnf
+
+
+OpenStack parameters and credentials
+------------------------------------
+
+Environment variables
+^^^^^^^^^^^^^^^^^^^^^
+Before running Yardstick (NSB Testing) it is necessary to export traffic
+generator libraries.
+
+::
+ source ~/.bash_profile
+
+Config yardstick conf
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+cp ./etc/yardstick/yardstick.conf.sample /etc/yardstick/yardstick.conf
+
+vi /etc/yardstick/yardstick.conf
+
+Config yardstick.conf
+::
+
+ [DEFAULT]
+ debug = True
+ dispatcher = influxdb
+
+ [dispatcher_influxdb]
+ timeout = 5
+ target = http://{YOUR_IP_HERE}:8086
+ db_name = yardstick
+ username = root
+ password = root
+
+ [nsb]
+ trex_path=/opt/nsb_bin/trex/scripts
+ bin_path=/opt/nsb_bin
+
+
+Config pod.yaml describing Topology
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Before executing Yardstick test cases, make sure that pod.yaml reflects the
+topology and update all the required fields.
+
+copy /etc/yardstick/nodes/pod.yaml.nsb.example to /etc/yardstick/nodes/pod.yaml
+
+Config pod.yaml
+::
+ nodes:
+ -
+ name: trafficgen_1
+ role: TrafficGen
+ ip: 1.1.1.1
+ user: root
+ password: r00t
+ interfaces:
+ xe0: # logical name from topology.yaml and vnfd.yaml
+ vpci: "0000:07:00.0"
+ driver: i40e # default kernel driver
+ dpdk_port_num: 0
+ local_ip: "152.16.100.20"
+ netmask: "255.255.255.0"
+ local_mac: "00:00:00:00:00:01"
+ xe1: # logical name from topology.yaml and vnfd.yaml
+ vpci: "0000:07:00.1"
+ driver: i40e # default kernel driver
+ dpdk_port_num: 1
+ local_ip: "152.16.40.20"
+ netmask: "255.255.255.0"
+ local_mac: "00:00.00:00:00:02"
+
+ -
+ name: vnf
+ role: vnf
+ ip: 1.1.1.2
+ user: root
+ password: r00t
+ host: 1.1.1.2 #BM - host == ip, virtualized env - Host - compute node
+ interfaces:
+ xe0: # logical name from topology.yaml and vnfd.yaml
+ vpci: "0000:07:00.0"
+ driver: i40e # default kernel driver
+ dpdk_port_num: 0
+ local_ip: "152.16.100.19"
+ netmask: "255.255.255.0"
+ local_mac: "00:00:00:00:00:03"
+
+ xe1: # logical name from topology.yaml and vnfd.yaml
+ vpci: "0000:07:00.1"
+ driver: i40e # default kernel driver
+ dpdk_port_num: 1
+ local_ip: "152.16.40.19"
+ netmask: "255.255.255.0"
+ local_mac: "00:00:00:00:00:04"
+ routing_table:
+ - network: "152.16.100.20"
+ netmask: "255.255.255.0"
+ gateway: "152.16.100.20"
+ if: "xe0"
+ - network: "152.16.40.20"
+ netmask: "255.255.255.0"
+ gateway: "152.16.40.20"
+ if: "xe1"
+ nd_route_tbl:
+ - network: "0064:ff9b:0:0:0:0:9810:6414"
+ netmask: "112"
+ gateway: "0064:ff9b:0:0:0:0:9810:6414"
+ if: "xe0"
+ - network: "0064:ff9b:0:0:0:0:9810:2814"
+ netmask: "112"
+ gateway: "0064:ff9b:0:0:0:0:9810:2814"
+ if: "xe1"
+
+Enable yardstick virtual environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Before executing yardstick test cases, make sure to activate yardstick
+python virtual environment
+
+::
+ source /opt/nsb_bin/yardstick_venv/bin/activate
+
+
+Examples and verifying the install
+----------------------------------
+
+It is recommended to verify that Yardstick was installed successfully
+by executing some simple commands and test samples. Before executing yardstick
+test cases make sure yardstick flavor and building yardstick-trusty-server
+image can be found in glance and openrc file is sourced. Below is an example
+invocation of yardstick help command and ping.py test sample:
+::
+
+ yardstick –h
+ yardstick task start samples/ping.yaml
+
+Each testing tool supported by Yardstick has a sample configuration file.
+These configuration files can be found in the **samples** directory.
+
+Default location for the output is ``/tmp/yardstick.out``.
+
+
+Run Yardstick - Network Service Testcases
+-----------------------------------------
+
+NS testing - using NSBperf CLI
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+::
+
+ source /opt/nsb_setup/yardstick_venv/bin/activate
+ PYTHONPATH: ". ~/.bash_profile"
+ cd <yardstick_repo>/yardstick/cmd
+ Execute command: ./NSPerf.py -h
+ ./NSBperf.py --vnf <selected vnf> --test <rfc test>
+ eg: ./NSBperf.py --vnf vpe --test tc_baremetal_rfc2544_ipv4_1flow_64B.yaml
+
+NS testing - using yardstick CLI
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+::
+
+ source /opt/nsb_setup/yardstick_venv/bin/activate
+ PYTHONPATH: ". ~/.bash_profile"
+ Go to test case forlder type we want to execute.
+ e.g. <yardstick repo>/samples/vnf_samples/nsut/<vnf>/
+ run: yardstick --debug task start <test_case.yaml>
diff --git a/docs/testing/user/userguide/09-installation.rst b/docs/testing/user/userguide/09-installation.rst
new file mode 100644
index 000000000..9c2082a27
--- /dev/null
+++ b/docs/testing/user/userguide/09-installation.rst
@@ -0,0 +1,401 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB, Huawei Technologies Co.,Ltd and others.
+
+Yardstick Installation
+======================
+
+Abstract
+--------
+
+Yardstick supports installation on Ubuntu 14.04 or via a Docker image. The
+installation procedure on Ubuntu 14.04 or via the docker image are detailed in
+the section below.
+
+To use Yardstick you should have access to an OpenStack environment, with at
+least Nova, Neutron, Glance, Keystone and Heat installed.
+
+The steps needed to run Yardstick are:
+
+1. Install Yardstick.
+2. Load OpenStack environment variables.
+3. Create a Neutron external network.
+4. Build Yardstick flavor and a guest image.
+5. Load the guest image into the OpenStack environment.
+6. Create the test configuration .yaml file.
+7. Run the test case.
+
+
+Prerequisites
+-------------
+
+The OPNFV deployment is out of the scope of this document but it can be
+found in http://artifacts.opnfv.org/opnfvdocs/colorado/docs/configguide/index.html.
+The OPNFV platform is considered as the System Under Test (SUT) in this
+document.
+
+Several prerequisites are needed for Yardstick:
+
+ #. A Jumphost to run Yardstick on
+ #. A Docker daemon shall be installed on the Jumphost
+ #. A public/external network created on the SUT
+ #. Connectivity from the Jumphost to the SUT public/external network
+
+WARNING: Connectivity from Jumphost is essential and it is of paramount
+importance to make sure it is working before even considering to install
+and run Yardstick. Make also sure you understand how your networking is
+designed to work.
+
+NOTE: **Jumphost** refers to any server which meets the previous
+requirements. Normally it is the same server from where the OPNFV
+deployment has been triggered previously.
+
+NOTE: If your Jumphost is operating behind a company http proxy and/or
+Firewall, please consult first the section `Proxy Support`_, towards
+the end of this document. The section details some tips/tricks which
+*may* be of help in a proxified environment.
+
+
+Installing Yardstick on Ubuntu 14.04
+------------------------------------
+
+.. _install-framework:
+
+You can install Yardstick framework directly on Ubuntu 14.04 or in an Ubuntu
+14.04 Docker image. No matter which way you choose to install Yardstick
+framework, the following installation steps are identical.
+
+If you choose to use the Ubuntu 14.04 Docker image, You can pull the Ubuntu
+14.04 Docker image from Docker hub:
+
+::
+
+ docker pull ubuntu:14.04
+
+Installing Yardstick framework
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Download source code and install python dependencies:
+
+::
+
+ git clone https://gerrit.opnfv.org/gerrit/yardstick
+ cd yardstick
+ ./install.sh
+
+
+Installing Yardstick using Docker
+---------------------------------
+
+Yardstick has a Docker image, this Docker image (**Yardstick-stable**)
+serves as a replacement for installing the Yardstick framework in a virtual
+environment (for example as done in :ref:`install-framework`).
+It is recommended to use this Docker image to run Yardstick test.
+
+Pulling the Yardstick Docker image
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. _dockerhub: https://hub.docker.com/r/opnfv/yardstick/
+
+Pull the Yardstick Docker image ('opnfv/yardstick') from the public dockerhub
+registry under the OPNFV account: [dockerhub_], with the following docker
+command::
+
+ docker pull opnfv/yardstick:stable
+
+After pulling the Docker image, check that it is available with the
+following docker command::
+
+ [yardsticker@jumphost ~]$ docker images
+ REPOSITORY TAG IMAGE ID CREATED SIZE
+ opnfv/yardstick stable a4501714757a 1 day ago 915.4 MB
+
+Run the Docker image:
+
+::
+
+ docker run --privileged=true -it opnfv/yardstick:stable /bin/bash
+
+In the container the Yardstick repository is located in the /home/opnfv/repos
+directory.
+
+
+OpenStack parameters and credentials
+------------------------------------
+
+Environment variables
+^^^^^^^^^^^^^^^^^^^^^
+Before running Yardstick it is necessary to export OpenStack environment variables
+from the OpenStack *openrc* file (using the ``source`` command) and export the
+external network name ``export EXTERNAL_NETWORK="external-network-name"``,
+the default name for the external network is ``net04_ext``.
+
+Credential environment variables in the *openrc* file have to include at least:
+
+* OS_AUTH_URL
+* OS_USERNAME
+* OS_PASSWORD
+* OS_TENANT_NAME
+
+A sample openrc file may look like this:
+
+* export OS_PASSWORD=console
+* export OS_TENANT_NAME=admin
+* export OS_AUTH_URL=http://172.16.1.222:35357/v2.0
+* export OS_USERNAME=admin
+* export OS_VOLUME_API_VERSION=2
+* export EXTERNAL_NETWORK=net04_ext
+
+
+Yardstick falvor and guest images
+---------------------------------
+
+Before executing Yardstick test cases, make sure that yardstick guest image and
+yardstick flavor are available in OpenStack.
+Detailed steps about creating yardstick flavor and building yardstick-trusty-server
+image can be found below.
+
+Yardstick-flavor
+^^^^^^^^^^^^^^^^
+Most of the sample test cases in Yardstick are using an OpenStack flavor called
+*yardstick-flavor* which deviates from the OpenStack standard m1.tiny flavor by the
+disk size - instead of 1GB it has 3GB. Other parameters are the same as in m1.tiny.
+
+Create yardstick-flavor:
+
+::
+
+ nova flavor-create yardstick-flavor 100 512 3 1
+
+
+.. _guest-image:
+
+Building a guest image
+^^^^^^^^^^^^^^^^^^^^^^
+Most of the sample test cases in Yardstick are using a guest image called
+*yardstick-trusty-server* which deviates from an Ubuntu Cloud Server image
+containing all the required tools to run test cases supported by Yardstick.
+Yardstick has a tool for building this custom image. It is necessary to have
+sudo rights to use this tool.
+
+Also you may need install several additional packages to use this tool, by
+follwing the commands below:
+
+::
+
+ apt-get update && apt-get install -y \
+ qemu-utils \
+ kpartx
+
+This image can be built using the following command while in the directory where
+Yardstick is installed (``~/yardstick`` if the framework is installed
+by following the commands above):
+
+::
+
+ export YARD_IMG_ARCH="amd64"
+ sudo echo "Defaults env_keep += \"YARD_IMG_ARCH\"" >> /etc/sudoers
+ sudo ./tools/yardstick-img-modify tools/ubuntu-server-cloudimg-modify.sh
+
+**Warning:** the script will create files by default in:
+``/tmp/workspace/yardstick`` and the files will be owned by root!
+
+If you are building this guest image in inside a docker container make sure the
+container is granted with privilege.
+
+The created image can be added to OpenStack using the ``glance image-create`` or
+via the OpenStack Dashboard.
+
+Example command:
+
+::
+
+ glance --os-image-api-version 1 image-create \
+ --name yardstick-image --is-public true \
+ --disk-format qcow2 --container-format bare \
+ --file /tmp/workspace/yardstick/yardstick-image.img
+
+Some Yardstick test cases use a Cirros image, you can find one at
+http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img
+
+
+Automatic flavor and image creation
+-----------------------------------
+Yardstick has a script for automatic creating yardstick flavor and building
+guest images. This script is mainly used in CI, but you can still use it in
+your local environment.
+
+Example command:
+
+::
+
+ export YARD_IMG_ARCH="amd64"
+ sudo echo "Defaults env_keep += \"YARD_IMG_ARCH\"" >> /etc/sudoers
+ source $YARDSTICK_REPO_DIR/tests/ci/load_images.sh
+
+
+Yardstick default key pair
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+Yardstick uses a SSH key pair to connect to the guest image. This key pair can
+be found in the ``resources/files`` directory. To run the ``ping-hot.yaml`` test
+sample, this key pair needs to be imported to the OpenStack environment.
+
+
+Examples and verifying the install
+----------------------------------
+
+It is recommended to verify that Yardstick was installed successfully
+by executing some simple commands and test samples. Before executing yardstick
+test cases make sure yardstick flavor and building yardstick-trusty-server
+image can be found in glance and openrc file is sourced. Below is an example
+invocation of yardstick help command and ping.py test sample:
+::
+
+ yardstick –h
+ yardstick task start samples/ping.yaml
+
+Each testing tool supported by Yardstick has a sample configuration file.
+These configuration files can be found in the **samples** directory.
+
+Default location for the output is ``/tmp/yardstick.out``.
+
+
+Deploy InfluxDB and Grafana locally
+------------------------------------
+
+.. pull docker images
+
+Pull docker images
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+ docker pull tutum/influxdb
+ docker pull grafana/grafana
+
+Run influxdb and config
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Run influxdb
+::
+
+ docker run -d --name influxdb \
+ -p 8083:8083 -p 8086:8086 --expose 8090 --expose 8099 \
+ tutum/influxdb
+ docker exec -it influxdb bash
+
+Config influxdb
+::
+
+ influx
+ >CREATE USER root WITH PASSWORD 'root' WITH ALL PRIVILEGES
+ >CREATE DATABASE yardstick;
+ >use yardstick;
+ >show MEASUREMENTS;
+
+Run grafana and config
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Run grafana
+::
+
+ docker run -d --name grafana -p 3000:3000 grafana/grafana
+
+Config grafana
+::
+
+ http://{YOUR_IP_HERE}:3000
+ log on using admin/admin and config database resource to be {YOUR_IP_HERE}:8086
+
+.. image:: images/Grafana_config.png
+ :width: 800px
+ :alt: Grafana data source configration
+
+Config yardstick conf
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+cp ./etc/yardstick/yardstick.conf.sample /etc/yardstick/yardstick.conf
+
+vi /etc/yardstick/yardstick.conf
+Config yardstick.conf
+::
+
+ [DEFAULT]
+ debug = True
+ dispatcher = influxdb
+
+ [dispatcher_influxdb]
+ timeout = 5
+ target = http://{YOUR_IP_HERE}:8086
+ db_name = yardstick
+ username = root
+ password = root
+
+Now you can run yardstick test cases and store the results in influxdb
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+Create a test suite for yardstick
+------------------------------------
+
+A test suite in yardstick is a yaml file which include one or more test cases.
+Yardstick is able to support running test suite task, so you can customize you
+own test suite and run it in one task.
+
+"tests/opnfv/test_suites" is where yardstick put ci test-suite. A typical test
+suite is like below:
+
+fuel_test_suite.yaml
+
+::
+
+ ---
+ # Fuel integration test task suite
+
+ schema: "yardstick:suite:0.1"
+
+ name: "fuel_test_suite"
+ test_cases_dir: "samples/"
+ test_cases:
+ -
+ file_name: ping.yaml
+ -
+ file_name: iperf3.yaml
+
+As you can see, there are two test cases in fuel_test_suite, the syntax is simple
+here, you must specify the schema and the name, then you just need to list the
+test cases in the tag "test_cases" and also mark their relative directory in the
+tag "test_cases_dir".
+
+Yardstick test suite also support constraints and task args for each test case.
+Here is another sample to show this, which is digested from one big test suite.
+
+os-nosdn-nofeature-ha.yaml
+
+::
+
+ ---
+
+ schema: "yardstick:suite:0.1"
+
+ name: "os-nosdn-nofeature-ha"
+ test_cases_dir: "tests/opnfv/test_cases/"
+ test_cases:
+ -
+ file_name: opnfv_yardstick_tc002.yaml
+ -
+ file_name: opnfv_yardstick_tc005.yaml
+ -
+ file_name: opnfv_yardstick_tc043.yaml
+ constraint:
+ installer: compass
+ pod: huawei-pod1
+ task_args:
+ huawei-pod1: '{"pod_info": "etc/yardstick/.../pod.yaml",
+ "host": "node4.LF","target": "node5.LF"}'
+
+As you can see in test case "opnfv_yardstick_tc043.yaml", there are two tags, "constraint" and
+"task_args". "constraint" is where you can specify which installer or pod it can be run in
+the ci environment. "task_args" is where you can specify the task arguments for each pod.
+
+All in all, to create a test suite in yardstick, you just need to create a suite yaml file
+and add test cases and constraint or task arguments if necessary.
+
diff --git a/docs/testing/user/userguide/10-yardstick_plugin.rst b/docs/testing/user/userguide/10-yardstick_plugin.rst
new file mode 100644
index 000000000..f16dedd02
--- /dev/null
+++ b/docs/testing/user/userguide/10-yardstick_plugin.rst
@@ -0,0 +1,144 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB, Huawei Technologies Co.,Ltd and others.
+
+===================================
+Installing a plug-in into yardstick
+===================================
+
+Abstract
+========
+
+Yardstick currently provides a ``plugin`` CLI command to support integration
+with other OPNFV testing projects. Below is an example invocation of yardstick
+plugin command and Storperf plug-in sample.
+
+
+Installing Storperf into yardstick
+==================================
+
+Storperf is delivered as a Docker container from
+https://hub.docker.com/r/opnfv/storperf/tags/.
+
+There are two possible methods for installation in your environment:
+
+* Run container on Jump Host
+* Run container in a VM
+
+In this introduction we will install Storperf on Jump Host.
+
+
+Step 0: Environment preparation
+>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+
+Running Storperf on Jump Host
+Requirements:
+
+* Docker must be installed
+* Jump Host must have access to the OpenStack Controller API
+* Jump Host must have internet connectivity for downloading docker image
+* Enough floating IPs must be available to match your agent count
+
+Before installing Storperf into yardstick you need to check your openstack
+environment and other dependencies:
+
+1. Make sure docker is installed.
+2. Make sure Keystone, Nova, Neutron, Glance, Heat are installed correctly.
+3. Make sure Jump Host have access to the OpenStack Controller API.
+4. Make sure Jump Host must have internet connectivity for downloading docker image.
+5. You need to know where to get basic openstack Keystone authorization info, such as
+ OS_PASSWORD, OS_TENANT_NAME, OS_AUTH_URL, OS_USERNAME.
+6. To run a Storperf container, you need to have OpenStack Controller environment
+ variables defined and passed to Storperf container. The best way to do this is to
+ put environment variables in a "storperf_admin-rc" file. The storperf_admin-rc
+ should include credential environment variables at least:
+
+* OS_AUTH_URL
+* OS_TENANT_ID
+* OS_TENANT_NAME
+* OS_PROJECT_NAME
+* OS_USERNAME
+* OS_PASSWORD
+* OS_REGION_NAME
+
+For this storperf_admin-rc file, during environment preparation a "prepare_storperf_admin-rc.sh"
+script can be used to generate it.
+::
+
+ #!/bin/bash
+ AUTH_URL=${OS_AUTH_URL}
+ USERNAME=${OS_USERNAME:-admin}
+ PASSWORD=${OS_PASSWORD:-console}
+ TENANT_NAME=${OS_TENANT_NAME:-admin}
+ VOLUME_API_VERSION=${OS_VOLUME_API_VERSION:-2}
+ PROJECT_NAME=${OS_PROJECT_NAME:-$TENANT_NAME}
+ TENANT_ID=`keystone tenant-get admin|grep 'id'|awk -F '|' '{print $3}'|sed -e 's/^[[:space:]]*//'`
+ rm -f ~/storperf_admin-rc
+ touch ~/storperf_admin-rc
+ echo "OS_AUTH_URL="$AUTH_URL >> ~/storperf_admin-rc
+ echo "OS_USERNAME="$USERNAME >> ~/storperf_admin-rc
+ echo "OS_PASSWORD="$PASSWORD >> ~/storperf_admin-rc
+ echo "OS_TENANT_NAME="$TENANT_NAME >> ~/storperf_admin-rc
+ echo "OS_VOLUME_API_VERSION="$VOLUME_API_VERSION >> ~/storperf_admin-rc
+ echo "OS_PROJECT_NAME="$PROJECT_NAME >> ~/storperf_admin-rc
+ echo "OS_TENANT_ID="$TENANT_ID >> ~/storperf_admin-rc
+
+
+Step 1: Plug-in configuration file preparation
+++++++++++++++++++++++++++++++++++++++++++++++
+
+To install a plug-in, first you need to prepare a plug-in configuration file in
+YAML format and store it in the "plugin" directory. The plugin configration file
+work as the input of yardstick "plugin" command. Below is the Storperf plug-in
+configuration file sample:
+::
+
+ ---
+ # StorPerf plugin configuration file
+ # Used for integration StorPerf into Yardstick as a plugin
+ schema: "yardstick:plugin:0.1"
+ plugins:
+ name: storperf
+ deployment:
+ ip: 192.168.23.2
+ user: root
+ password: root
+
+In the plug-in configuration file, you need to specify the plug-in name and the
+plug-in deployment info, including node ip, node login username and password.
+Here the Storperf will be installed on IP 192.168.23.2 which is the Jump Host
+in my local environment.
+
+Step 2: Plug-in install/remove scripts preparation
+++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Under "yardstick/resource/scripts directory", there are two folders: a "install"
+folder and a "remove" folder. You need to store the plug-in install/remove script
+in these two folders respectively.
+
+The detailed installation or remove operation should de defined in these two scripts.
+The name of both install and remove scripts should match the plugin-in name that you
+specified in the plug-in configuration file.
+For example, the install and remove scripts for Storperf are both named to "storperf.bash".
+
+
+Step 3: Install and remove Storperf
++++++++++++++++++++++++++++++++++++
+
+To install Storperf, simply execute the following command
+::
+
+ # Install Storperf
+ yardstick plugin install plugin/storperf.yaml
+
+removing Storperf from yardstick
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To remove Storperf, simply execute the following command
+::
+
+ # Remove Storperf
+ yardstick plugin remove plugin/storperf.yaml
+
+What yardstick plugin command does is using the username and password to log into the deployment target and then execute the corresponding install or remove script.
diff --git a/docs/testing/user/userguide/11-result-store-InfluxDB.rst b/docs/testing/user/userguide/11-result-store-InfluxDB.rst
new file mode 100644
index 000000000..a0bb48a80
--- /dev/null
+++ b/docs/testing/user/userguide/11-result-store-InfluxDB.rst
@@ -0,0 +1,86 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, 2016 Huawei Technologies Co.,Ltd and others.
+
+==============================================
+Store Other Project's Test Results in InfluxDB
+==============================================
+
+Abstract
+========
+
+.. _Framework: https://wiki.opnfv.org/download/attachments/6827660/wiki.png?version=1&modificationDate=1470298075000&api=v2
+
+This chapter illustrates how to run plug-in test cases and store test results
+into community's InfluxDB. The framework is shown in Framework_.
+
+
+.. image:: images/InfluxDB_store.png
+ :width: 800px
+ :alt: Store Other Project's Test Results in InfluxDB
+
+Store Storperf Test Results into Community's InfluxDB
+=====================================================
+
+.. _Influxdb: https://git.opnfv.org/cgit/yardstick/tree/yardstick/dispatcher/influxdb.py
+.. _Mingjiang: limingjiang@huawei.com
+.. _Visual: https://wiki.opnfv.org/download/attachments/6827660/tc074.PNG?version=1&modificationDate=1470298075000&api=v2
+.. _Login: http://testresults.opnfv.org/grafana/login
+
+As shown in Framework_, there are two ways to store Storperf test results
+into community's InfluxDB:
+
+1. Yardstick asks Storperf to run the test case. After the test case is
+ completed, Yardstick reads test results via ReST API from Storperf and
+ posts test data to the influxDB.
+
+2. Additionally, Storperf can run tests by itself and post the test result
+ directly to the InfluxDB. The method for posting data directly to influxDB
+ will be supported in the future.
+
+Our plan is to support rest-api in D release so that other testing projects can
+call the rest-api to use yardstick dispatcher service to push data to yardstick's
+influxdb database.
+
+For now, influxdb only support line protocol, and the json protocol is deprecated.
+
+Take ping test case for example, the raw_result is json format like this:
+::
+
+ "benchmark": {
+ "timestamp": 1470315409.868095,
+ "errors": "",
+ "data": {
+ "rtt": {
+ "ares": 1.125
+ }
+ },
+ "sequence": 1
+ },
+ "runner_id": 2625
+ }
+
+With the help of "influxdb_line_protocol", the json is transform to like below as a line string:
+::
+
+ 'ping,deploy_scenario=unknown,host=athena.demo,installer=unknown,pod_name=unknown,
+ runner_id=2625,scenarios=Ping,target=ares.demo,task_id=77755f38-1f6a-4667-a7f3-
+ 301c99963656,version=unknown rtt.ares=1.125 1470315409868094976'
+
+So, for data output of json format, you just need to transform json into line format and call
+influxdb api to post the data into the database. All this function has been implemented in Influxdb_.
+If you need support on this, please contact Mingjiang_.
+::
+
+ curl -i -XPOST 'http://104.197.68.199:8086/write?db=yardstick' --
+ data-binary 'ping,deploy_scenario=unknown,host=athena.demo,installer=unknown, ...'
+
+Grafana will be used for visualizing the collected test data, which is shown in Visual_. Grafana
+can be accessed by Login_.
+
+
+.. image:: images/results_visualization.png
+ :width: 800px
+ :alt: results visualization
+
diff --git a/docs/testing/user/userguide/12-grafana.rst b/docs/testing/user/userguide/12-grafana.rst
new file mode 100644
index 000000000..416857b71
--- /dev/null
+++ b/docs/testing/user/userguide/12-grafana.rst
@@ -0,0 +1,119 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) 2016 Huawei Technologies Co.,Ltd and others
+
+=================
+Grafana dashboard
+=================
+
+
+Abstract
+========
+
+This chapter describes the Yardstick grafana dashboard. The Yardstick grafana
+dashboard can be found here: http://testresults.opnfv.org/grafana/
+
+
+.. image:: images/login.png
+ :width: 800px
+ :alt: Yardstick grafana dashboard
+
+
+Public access
+=============
+
+Yardstick provids a public account for accessing to the dashboard. The username
+and password are both set to ‘opnfv’.
+
+
+Testcase dashboard
+==================
+
+For each test case, there is a dedicated dashboard. Shown here is the dashboard
+of TC002.
+
+
+.. image:: images/TC002.png
+ :width: 800px
+ :alt:TC002 dashboard
+
+For each test case dashboard. On the top left, we have a dashboard selection,
+you can switch to different test cases using this pull-down menu.
+
+Underneath, we have a pod and scenario selection.
+All the pods and scenarios that have ever published test data to the InfluxDB
+will be shown here.
+
+You can check multiple pods or scenarios.
+
+For each test case, we have a short description and a link to detailed test
+case information in Yardstick user guide.
+
+Underneath, it is the result presentation section.
+You can use the time period selection on the top right corner to zoom in or
+zoom out the chart.
+
+
+Administration access
+=====================
+
+For a user with administration rights it is easy to update and save any
+dashboard configuration. Saved updates immediately take effect and become live.
+This may cause issues like:
+
+- Changes and updates made to the live configuration in Grafana can compromise
+ existing Grafana content in an unwanted, unpredicted or incompatible way.
+ Grafana as such is not version controlled, there exists one single Grafana
+ configuration per dashboard.
+- There is a risk several people can disturb each other when doing updates to
+ the same Grafana dashboard at the same time.
+
+Any change made by administrator should be careful.
+
+
+Add a dashboard into yardstick grafana
+======================================
+
+Due to security concern, users that using the public opnfv account are not able
+to edit the yardstick grafana directly.It takes a few more steps for a
+non-yardstick user to add a custom dashboard into yardstick grafana.
+
+There are 6 steps to go.
+
+
+.. image:: images/add.png
+ :width: 800px
+ :alt: Add a dashboard into yardstick grafana
+
+
+1. You need to build a local influxdb and grafana, so you can do the work
+ locally. You can refer to How to deploy InfluxDB and Grafana locally wiki
+ page about how to do this.
+
+2. Once step one is done, you can fetch the existing grafana dashboard
+ configuration file from the yardstick repository and import it to your local
+ grafana. After import is done, you grafana dashboard will be ready to use
+ just like the community’s dashboard.
+
+3. The third step is running some test cases to generate test results and
+ publishing it to your local influxdb.
+
+4. Now you have some data to visualize in your dashboard. In the fourth step,
+ it is time to create your own dashboard. You can either modify an existing
+ dashboard or try to create a new one from scratch. If you choose to modify
+ an existing dashboard then in the curtain menu of the existing dashboard do
+ a "Save As..." into a new dashboard copy instance, and then continue doing
+ all updates and saves within the dashboard copy.
+
+5. When finished with all Grafana configuration changes in this temporary
+ dashboard then chose "export" of the updated dashboard copy into a JSON file
+ and put it up for review in Gerrit, in file /yardstick/dashboard/Yardstick-TCxxx-yyyyyyyyyyyyy.
+ For instance a typical default name of the file would be "Yardstick-TC001 Copy-1234567891234".
+
+6. Once you finish your dashboard, the next step is exporting the configuration
+ file and propose a patch into Yardstick. Yardstick team will review and
+ merge it into Yardstick repository. After approved review Yardstick team
+ will do an "import" of the JSON file and also a "save dashboard" as soon as
+ possible to replace the old live dashboard configuration.
+
diff --git a/docs/testing/user/userguide/13-list-of-tcs.rst b/docs/testing/user/userguide/13-list-of-tcs.rst
new file mode 100644
index 000000000..1b5806cd9
--- /dev/null
+++ b/docs/testing/user/userguide/13-list-of-tcs.rst
@@ -0,0 +1,129 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+====================
+Yardstick Test Cases
+====================
+
+Abstract
+========
+
+This chapter lists available Yardstick test cases.
+Yardstick test cases are divided in two main categories:
+
+* *Generic NFVI Test Cases* - Test Cases developed to realize the methodology
+described in :doc:`02-methodology`
+
+* *OPNFV Feature Test Cases* - Test Cases developed to verify one or more
+aspect of a feature delivered by an OPNFV Project, including the test cases
+developed for the :term:`VTC`.
+
+Generic NFVI Test Case Descriptions
+===================================
+
+.. toctree::
+ :maxdepth: 1
+
+ opnfv_yardstick_tc001.rst
+ opnfv_yardstick_tc002.rst
+ opnfv_yardstick_tc004.rst
+ opnfv_yardstick_tc005.rst
+ opnfv_yardstick_tc008.rst
+ opnfv_yardstick_tc009.rst
+ opnfv_yardstick_tc010.rst
+ opnfv_yardstick_tc011.rst
+ opnfv_yardstick_tc012.rst
+ opnfv_yardstick_tc014.rst
+ opnfv_yardstick_tc024.rst
+ opnfv_yardstick_tc037.rst
+ opnfv_yardstick_tc038.rst
+ opnfv_yardstick_tc042.rst
+ opnfv_yardstick_tc043.rst
+ opnfv_yardstick_tc044.rst
+ opnfv_yardstick_tc055.rst
+ opnfv_yardstick_tc061.rst
+ opnfv_yardstick_tc063.rst
+ opnfv_yardstick_tc069.rst
+ opnfv_yardstick_tc070.rst
+ opnfv_yardstick_tc071.rst
+ opnfv_yardstick_tc072.rst
+ opnfv_yardstick_tc073.rst
+ opnfv_yardstick_tc075.rst
+ opnfv_yardstick_tc076.rst
+
+OPNFV Feature Test Cases
+========================
+
+H A
+---
+
+.. toctree::
+ :maxdepth: 1
+
+ opnfv_yardstick_tc019.rst
+ opnfv_yardstick_tc025.rst
+ opnfv_yardstick_tc045.rst
+ opnfv_yardstick_tc046.rst
+ opnfv_yardstick_tc047.rst
+ opnfv_yardstick_tc048.rst
+ opnfv_yardstick_tc049.rst
+ opnfv_yardstick_tc050.rst
+ opnfv_yardstick_tc051.rst
+ opnfv_yardstick_tc052.rst
+ opnfv_yardstick_tc053.rst
+ opnfv_yardstick_tc054.rst
+
+IPv6
+----
+
+.. toctree::
+ :maxdepth: 1
+
+ opnfv_yardstick_tc027.rst
+
+KVM
+---
+
+.. toctree::
+ :maxdepth: 1
+
+ opnfv_yardstick_tc028.rst
+
+Parser
+------
+
+.. toctree::
+ :maxdepth: 1
+
+ opnfv_yardstick_tc040.rst
+
+ StorPerf
+-----------
+
+.. toctree::
+ :maxdepth: 1
+
+ opnfv_yardstick_tc074.rst
+
+virtual Traffic Classifier
+--------------------------
+
+.. toctree::
+ :maxdepth: 1
+
+ opnfv_yardstick_tc006.rst
+ opnfv_yardstick_tc007.rst
+ opnfv_yardstick_tc020.rst
+ opnfv_yardstick_tc021.rst
+
+Templates
+=========
+
+.. toctree::
+ :maxdepth: 1
+
+ testcase_description_v2_template
+ Yardstick_task_templates
+
diff --git a/docs/testing/user/userguide/Yardstick_task_templates.rst b/docs/testing/user/userguide/Yardstick_task_templates.rst
new file mode 100755
index 000000000..e8130dd2a
--- /dev/null
+++ b/docs/testing/user/userguide/Yardstick_task_templates.rst
@@ -0,0 +1,160 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+Task Template Syntax
+====================
+
+Basic template syntax
+---------------------
+A nice feature of the input task format used in Yardstick is that it supports
+the template syntax based on Jinja2.
+This turns out to be extremely useful when, say, you have a fixed structure of
+your task but you want to parameterize this task in some way.
+For example, imagine your input task file (task.yaml) runs a set of Ping
+scenarios:
+
+::
+
+ # Sample benchmark task config file
+ # measure network latency using ping
+ schema: "yardstick:task:0.1"
+
+ scenarios:
+ -
+ type: Ping
+ options:
+ packetsize: 200
+ host: athena.demo
+ target: ares.demo
+
+ runner:
+ type: Duration
+ duration: 60
+ interval: 1
+
+ sla:
+ max_rtt: 10
+ action: monitor
+
+ context:
+ ...
+
+Let's say you want to run the same set of scenarios with the same runner/
+context/sla, but you want to try another packetsize to compare the performance.
+The most elegant solution is then to turn the packetsize name into a template
+variable:
+
+::
+
+ # Sample benchmark task config file
+ # measure network latency using ping
+
+ schema: "yardstick:task:0.1"
+ scenarios:
+ -
+ type: Ping
+ options:
+ packetsize: {{packetsize}}
+ host: athena.demo
+ target: ares.demo
+
+ runner:
+ type: Duration
+ duration: 60
+ interval: 1
+
+ sla:
+ max_rtt: 10
+ action: monitor
+
+ context:
+ ...
+
+and then pass the argument value for {{packetsize}} when starting a task with
+this configuration file.
+Yardstick provides you with different ways to do that:
+
+1.Pass the argument values directly in the command-line interface (with either
+a JSON or YAML dictionary):
+
+::
+
+ yardstick task start samples/ping-template.yaml
+ --task-args'{"packetsize":"200"}'
+
+2.Refer to a file that specifies the argument values (JSON/YAML):
+
+::
+
+ yardstick task start samples/ping-template.yaml --task-args-file args.yaml
+
+Using the default values
+------------------------
+Note that the Jinja2 template syntax allows you to set the default values for
+your parameters.
+With default values set, your task file will work even if you don't
+parameterize it explicitly while starting a task.
+The default values should be set using the {% set ... %} clause (task.yaml).
+For example:
+
+::
+
+ # Sample benchmark task config file
+ # measure network latency using ping
+ schema: "yardstick:task:0.1"
+ {% set packetsize = packetsize or "100" %}
+ scenarios:
+ -
+ type: Ping
+ options:
+ packetsize: {{packetsize}}
+ host: athena.demo
+ target: ares.demo
+
+ runner:
+ type: Duration
+ duration: 60
+ interval: 1
+ ...
+
+If you don't pass the value for {{packetsize}} while starting a task, the
+default one will be used.
+
+Advanced templates
+------------------
+
+Yardstick makes it possible to use all the power of Jinja2 template syntax,
+including the mechanism of built-in functions.
+As an example, let us make up a task file that will do a block storage
+performance test.
+The input task file (fio-template.yaml) below uses the Jinja2 for-endfor
+construct to accomplish that:
+
+::
+
+ #Test block sizes of 4KB, 8KB, 64KB, 1MB
+ #Test 5 workloads: read, write, randwrite, randread, rw
+ schema: "yardstick:task:0.1"
+
+ scenarios:
+ {% for bs in ['4k', '8k', '64k', '1024k' ] %}
+ {% for rw in ['read', 'write', 'randwrite', 'randread', 'rw' ] %}
+ -
+ type: Fio
+ options:
+ filename: /home/ubuntu/data.raw
+ bs: {{bs}}
+ rw: {{rw}}
+ ramp_time: 10
+ host: fio.demo
+ runner:
+ type: Duration
+ duration: 60
+ interval: 60
+
+ {% endfor %}
+ {% endfor %}
+ context
+ ...
diff --git a/docs/testing/user/userguide/comp-intro.rst b/docs/testing/user/userguide/comp-intro.rst
new file mode 100644
index 000000000..ee68226ad
--- /dev/null
+++ b/docs/testing/user/userguide/comp-intro.rst
@@ -0,0 +1,37 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+=========
+Yardstick
+=========
+
+.. _Yardstick: https://wiki.opnfv.org/yardstick
+.. _Presentation: https://wiki.opnfv.org/_media/opnfv_summit_-_yardstick_project.pdf
+.. _NFV-TST001: https://docbox.etsi.org/ISG/NFV/Open/Drafts/TST001_-_Pre-deployment_Validation/
+.. _Yardsticktst: https://wiki.opnfv.org/_media/opnfv_summit_-_bridging_opnfv_and_etsi.pdf
+
+The project's goal is to verify infrastructure compliance, from the perspective
+of a Virtual Network Function (VNF).
+
+The Project's scope is the development of a test framework, *Yardstick*, test
+cases and test stimuli to enable Network Function Virtualization Infrastructure
+(NFVI) verification.
+
+In OPNFV Brahmaputra release, generic test cases covering aspects of the
+metrics in the document ETSI GS NFV-TST001_, "Pre-deployment Testing; Report on
+Validation of NFV Environments and Services" are available; further OPNFV
+releases will provide extended testing of these metrics.
+
+The Project also includes a sample VNF, the Virtual Traffic Classifier (VTC)
+and its experimental framework, *ApexLake*.
+
+*Yardstick* is used in OPNFV for verifying the OPNFV infrastructure and some of
+the OPNFV features. The *Yardstick* framework is deployed in several OPNFV
+community labs. It is *installer*, *infrastructure* and *application*
+independent.
+
+
+.. seealso:: This Presentation_ for an overview of *Yardstick* and
+ Yardsticktst_ for material on alignment ETSI TST001 and Yardstick.
diff --git a/docs/testing/user/userguide/glossary.rst b/docs/testing/user/userguide/glossary.rst
new file mode 100644
index 000000000..f8ff41887
--- /dev/null
+++ b/docs/testing/user/userguide/glossary.rst
@@ -0,0 +1,65 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+========
+Glossary
+========
+
+.. glossary::
+ :sorted:
+
+ API
+ Application Programming Interface
+
+ DPI
+ Deep Packet Inspection
+
+ DPDK
+ Data Plane Development Kit
+
+ DSCP
+ Differentiated Services Code Point
+
+ IGMP
+ Internet Group Management Protocol
+
+ IOPS
+ Input/Output Operations Per Second
+
+ NIC
+ Network Interface Controller
+
+ PBFS
+ Packet Based per Flow State
+
+ QoS
+ Quality of Service
+
+ VLAN
+ Virtual LAN
+
+ VM
+ Virtual Machine
+
+ VNF
+ Virtual Network Function
+
+ VNFC
+ Virtual Network Function Component
+
+ NFVI
+ Network Function Virtualization Infrastructure
+
+ SR-IOV
+ Single Root IO Virtualization
+
+ SUT
+ System Under Test
+
+ ToS
+ Type of Service
+
+ VTC
+ Virtual Traffic Classifier
diff --git a/docs/testing/user/userguide/images/Deployment.png b/docs/testing/user/userguide/images/Deployment.png
new file mode 100755
index 000000000..aca5670cd
--- /dev/null
+++ b/docs/testing/user/userguide/images/Deployment.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/Grafana_config.png b/docs/testing/user/userguide/images/Grafana_config.png
new file mode 100644
index 000000000..cb63098dc
--- /dev/null
+++ b/docs/testing/user/userguide/images/Grafana_config.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/InfluxDB_store.png b/docs/testing/user/userguide/images/InfluxDB_store.png
new file mode 100644
index 000000000..1770fd255
--- /dev/null
+++ b/docs/testing/user/userguide/images/InfluxDB_store.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/Logical_view.png b/docs/testing/user/userguide/images/Logical_view.png
new file mode 100644
index 000000000..cdb805448
--- /dev/null
+++ b/docs/testing/user/userguide/images/Logical_view.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/TC002.png b/docs/testing/user/userguide/images/TC002.png
new file mode 100644
index 000000000..89154efcc
--- /dev/null
+++ b/docs/testing/user/userguide/images/TC002.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/Use_case.png b/docs/testing/user/userguide/images/Use_case.png
new file mode 100644
index 000000000..acd52f526
--- /dev/null
+++ b/docs/testing/user/userguide/images/Use_case.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/add.png b/docs/testing/user/userguide/images/add.png
new file mode 100644
index 000000000..a88a1b146
--- /dev/null
+++ b/docs/testing/user/userguide/images/add.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/login.png b/docs/testing/user/userguide/images/login.png
new file mode 100644
index 000000000..045e010e4
--- /dev/null
+++ b/docs/testing/user/userguide/images/login.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/results_visualization.png b/docs/testing/user/userguide/images/results_visualization.png
new file mode 100644
index 000000000..cd092808b
--- /dev/null
+++ b/docs/testing/user/userguide/images/results_visualization.png
Binary files differ
diff --git a/docs/testing/user/userguide/images/test_execution_flow.png b/docs/testing/user/userguide/images/test_execution_flow.png
new file mode 100644
index 000000000..c20a931a4
--- /dev/null
+++ b/docs/testing/user/userguide/images/test_execution_flow.png
Binary files differ
diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst
new file mode 100644
index 000000000..1b963af61
--- /dev/null
+++ b/docs/testing/user/userguide/index.rst
@@ -0,0 +1,27 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+==================
+Performance Testing User Guide (Yardstick)
+==================
+
+.. toctree::
+ :maxdepth: 2
+
+ 01-introduction
+ 02-methodology
+ 03-architecture
+ 04-vtc-overview
+ 05-apexlake_installation
+ 06-apexlake_api
+ 07-nsb-overview
+ 08-nsb_installation
+ 09-installation
+ 10-yardstick_plugin
+ 11-result-store-InfluxDB
+ 12-grafana
+ 13-list-of-tcs
+ glossary
+ references
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc001.rst b/docs/testing/user/userguide/opnfv_yardstick_tc001.rst
new file mode 100644
index 000000000..b53c508a6
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc001.rst
@@ -0,0 +1,133 @@
+s work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC001
+*************************************
+
+.. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+
++-----------------------------------------------------------------------------+
+|Network Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC001_NETWORK PERFORMANCE |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of flows and throughput |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC001 is to evaluate the IaaS network |
+| | performance with regards to flows and throughput, such as if |
+| | and how different amounts of flows matter for the throughput |
+| | between hosts on different compute blades. Typically e.g. |
+| | the performance of a vSwitch depends on the number of flows |
+| | running through it. Also performance of other equipment or |
+| | entities can depend on the number of flows or the packet |
+| | sizes used. |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | pktgen |
+| | |
+| | Linux packet generator is a tool to generate packets at very |
+| | high speed in the kernel. pktgen is mainly used to drive and |
+| | LAN equipment test network. pktgen supports multi threading. |
+| | To generate random MAC address, IP address, port number UDP |
+| | packets, pktgen uses multiple CPU processors in the |
+| | different PCI bus (PCI, PCIe bus) with Gigabit Ethernet |
+| | tested (pktgen performance depends on the CPU processing |
+| | speed, memory delay, PCI bus speed hardware parameters), |
+| | Transmit data rate can be even larger than 10GBit/s. Visible |
+| | can satisfy most card test requirements. |
+| | |
+| | (Pktgen is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Docker |
+| | image. |
+| | As an example see the /yardstick/tools/ directory for how |
+| | to generate a Linux image with pktgen included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|test | This test case uses Pktgen to generate packet flow between |
+|description | two hosts for simulating network workloads on the SUT. |
+| | |
++--------------+--------------------------------------------------------------+
+|traffic | An IP table is setup on server to monitor for received |
+|profile | packets. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc001.yaml |
+| | |
+| | Packet size is set to 60 bytes. |
+| | Number of ports: 10, 50, 100, 500 and 1000, where each |
+| | runs for 20 seconds. The whole sequence is run twice |
+| | The client and server are distributed on different hardware. |
+| | |
+| | For SLA max_ppm is set to 1000. The amount of configured |
+| | ports map to between 110 up to 1001000 flows, respectively. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * packet sizes; |
+| | * amount of flows; |
+| | * test duration. |
+| | |
+| | Default values exist. |
+| | |
+| | SLA (optional): max_ppm: The number of packets per million |
+| | packets sent that are acceptable to loose, not received. |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is used for generating high network |
+| | throughput to simulate certain workloads on the SUT. Hence |
+| | it should work with other test cases. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | pktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with pktgen included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | Two host VMs are booted, as server and client. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the server VM by using ssh. |
+| | 'pktgen_benchmark' bash script is copyied from Jump Host to |
+| | the server VM via the ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | An IP table is setup on server to monitor for received |
+| | packets. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | pktgen is invoked to generate packet flow between two server |
+| | and client for simulating network workloads on the SUT. |
+| | Results are processed and checked against the SLA. Logs are |
+| | produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 5 | Two host VMs are deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc002.rst b/docs/testing/user/userguide/opnfv_yardstick_tc002.rst
new file mode 100644
index 000000000..c98780fd5
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc002.rst
@@ -0,0 +1,126 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC002
+*************************************
+
+.. _cirros-image: https://download.cirros-cloud.net
+.. _Ping: https://linux.die.net/man/8/ping
+
++-----------------------------------------------------------------------------+
+|Network Latency |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC002_NETWORK LATENCY |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | RTT (Round Trip Time) |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC002 is to do a basic verification that |
+| | network latency is within acceptable boundaries when packets |
+| | travel between hosts located on same or different compute |
+| | blades. |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | ping |
+| | |
+| | Ping is a computer network administration software utility |
+| | used to test the reachability of a host on an Internet |
+| | Protocol (IP) network. It measures the round-trip time for |
+| | packet sent from the originating host to a destination |
+| | computer that are echoed back to the source. |
+| | |
+| | Ping is normally part of any Linux distribution, hence it |
+| | doesn't need to be installed. It is also part of the |
+| | Yardstick Docker image. |
+| | (For example also a Cirros image can be downloaded from |
+| | cirros-image_, it includes ping) |
+| | |
++--------------+--------------------------------------------------------------+
+|test topology | Ping packets (ICMP protocol's mandatory ECHO_REQUEST |
+| | datagram) are sent from host VM to target VM(s) to elicit |
+| | ICMP ECHO_RESPONSE. |
+| | |
+| | For one host VM there can be multiple target VMs. |
+| | Host VM and target VM(s) can be on same or different compute |
+| | blades. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc002.yaml |
+| | |
+| | Packet size 100 bytes. Test duration 60 seconds. |
+| | One ping each 10 seconds. Test is iterated two times. |
+| | SLA RTT is set to maximum 10 ms. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | This test case can be configured with different: |
+| | |
+| | * packet sizes; |
+| | * burst sizes; |
+| | * ping intervals; |
+| | * test durations; |
+| | * test iterations. |
+| | |
+| | Default values exist. |
+| | |
+| | SLA is optional. The SLA in this test case serves as an |
+| | example. Considerably lower RTT is expected, and also normal |
+| | to achieve in balanced L2 environments. However, to cover |
+| | most configurations, both bare metal and fully virtualized |
+| | ones, this value should be possible to achieve and |
+| | acceptable for black box testing. Many real time |
+| | applications start to suffer badly if the RTT time is higher |
+| | than this. Some may suffer bad also close to this RTT, while |
+| | others may not suffer at all. It is a compromise that may |
+| | have to be tuned for different configuration purposes. |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is one of Yardstick's generic test. Thus it |
+| | is runnable on most of the scenarios. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Ping_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image (cirros-image) needs to be installed |
+|conditions | into Glance with ping included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | Two host VMs are booted, as server and client. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the server VM by using ssh. |
+| | 'ping_benchmark' bash script is copyied from Jump Host to |
+| | the server VM via the ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | Ping is invoked. Ping packets are sent from server VM to |
+| | client VM. RTT results are calculated and checked against |
+| | the SLA. Logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | Two host VMs are deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Test should not PASS if any RTT is above the optional SLA |
+| | value, or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc004.rst b/docs/testing/user/userguide/opnfv_yardstick_tc004.rst
new file mode 100644
index 000000000..3554b3826
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc004.rst
@@ -0,0 +1,110 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC004
+*************************************
+
+.. _cachestat: https://github.com/brendangregg/perf-tools/tree/master/fs
+
++-----------------------------------------------------------------------------+
+|Cache Utilization |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC004_CACHE Utilization |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | cache hit, cache miss, hit/miss ratio, buffer size and page |
+| | cache size |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC004 is to evaluate the IaaS compute |
+| | capability with regards to cache utilization.This test case |
+| | should be run in parallel with other Yardstick test cases |
+| | and not run as a stand-alone test case. |
+| | |
+| | This test case measures cache usage statistics, including |
+| | cache hit, cache miss, hit ratio, buffer cache size and page |
+| | cache size, with some wokloads runing on the infrastructure. |
+| | Both average and maximun values are collected. |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | cachestat |
+| | |
+| | cachestat is a tool using Linux ftrace capabilities for |
+| | showing Linux page cache hit/miss statistics. |
+| | |
+| | (cachestat is not always part of a Linux distribution, hence |
+| | it needs to be installed. As an example see the |
+| | /yardstick/tools/ directory for how to generate a Linux |
+| | image with cachestat included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|test | cachestat test is invoked in a host VM on a compute blade, |
+|description | cachestat test requires some other test cases running in the |
+| | host to stimulate workload. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | File: cachestat.yaml (in the 'samples' directory) |
+| | |
+| | Interval is set 1. Test repeat, pausing every 1 seconds |
+| | in-between. |
+| | Test durarion is set to 60 seconds. |
+| | |
+| | SLA is not available in this test case. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * interval; |
+| | * runner Duration. |
+| | |
+| | Default values exist. |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is one of Yardstick's generic test. Thus it |
+| | is runnable on most of the scenarios. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | cachestat_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with cachestat included in the image. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | A host VM with cachestat installed is booted. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the host VM by using ssh. |
+| | 'cache_stat' bash script is copyied from Jump Host to |
+| | the server VM via the ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | 'cache_stat' script is invoked. Raw cache usage statistics |
+| | are collected and filtrated. Average and maximum values are |
+| | calculated and recorded. Logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | The host VM is deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. Cache utilization results are collected and stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc005.rst b/docs/testing/user/userguide/opnfv_yardstick_tc005.rst
new file mode 100644
index 000000000..1c2d71d81
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc005.rst
@@ -0,0 +1,125 @@
+. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC005
+*************************************
+
+.. _fio: http://bluestop.org/files/fio/HOWTO.txt
+
++-----------------------------------------------------------------------------+
+|Storage Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC005_STORAGE PERFORMANCE |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | IOPS (Average IOs performed per second), |
+| | Throughput (Average disk read/write bandwidth rate), |
+| | Latency (Average disk read/write latency) |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC005 is to evaluate the IaaS storage |
+| | performance with regards to IOPS, throughput and latency. |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | fio |
+| | |
+| | fio is an I/O tool meant to be used both for benchmark and |
+| | stress/hardware verification. It has support for 19 |
+| | different types of I/O engines (sync, mmap, libaio, |
+| | posixaio, SG v3, splice, null, network, syslet, guasi, |
+| | solarisaio, and more), I/O priorities (for newer Linux |
+| | kernels), rate I/O, forked or threaded jobs, and much more. |
+| | |
+| | (fio is not always part of a Linux distribution, hence it |
+| | needs to be installed. As an example see the |
+| | /yardstick/tools/ directory for how to generate a Linux |
+| | image with fio included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|test | fio test is invoked in a host VM on a compute blade, a job |
+|description | file as well as parameters are passed to fio and fio will |
+| | start doing what the job file tells it to do. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc005.yaml |
+| | |
+| | IO types is set to read, write, randwrite, randread, rw. |
+| | IO block size is set to 4KB, 64KB, 1024KB. |
+| | fio is run for each IO type and IO block size scheme, |
+| | each iteration runs for 30 seconds (10 for ramp time, 20 for |
+| | runtime). |
+| | |
+| | For SLA, minimum read/write iops is set to 100, |
+| | minimum read/write throughput is set to 400 KB/s, |
+| | and maximum read/write latency is set to 20000 usec. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | This test case can be configured with different: |
+| | |
+| | * IO types; |
+| | * IO block size; |
+| | * IO depth; |
+| | * ramp time; |
+| | * test duration. |
+| | |
+| | Default values exist. |
+| | |
+| | SLA is optional. The SLA in this test case serves as an |
+| | example. Considerably higher throughput and lower latency |
+| | are expected. However, to cover most configurations, both |
+| | baremetal and fully virtualized ones, this value should be |
+| | possible to achieve and acceptable for black box testing. |
+| | Many heavy IO applications start to suffer badly if the |
+| | read/write bandwidths are lower than this. |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is one of Yardstick's generic test. Thus it |
+| | is runnable on most of the scenarios. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | fio_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with fio included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | A host VM with fio installed is booted. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the host VM by using ssh. |
+| | 'fio_benchmark' bash script is copyied from Jump Host to |
+| | the host VM via the ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | 'fio_benchmark' script is invoked. Simulated IO operations |
+| | are started. IOPS, disk read/write bandwidth and latency are |
+| | recorded and checked against the SLA. Logs are produced and |
+| | stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | The host VM is deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc006.rst b/docs/testing/user/userguide/opnfv_yardstick_tc006.rst
new file mode 100644
index 000000000..2ccb417c1
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc006.rst
@@ -0,0 +1,144 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Intel Corporation and others.
+
+*************************************
+Yardstick Test Case Description TC006
+*************************************
+
+.. _DPDKpktgen: https://github.com/Pktgen/Pktgen-DPDK/
+.. _rfc2544: https://www.ietf.org/rfc/rfc2544.txt
+
++-----------------------------------------------------------------------------+
+|Network Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC006_Virtual Traffic Classifier Data Plane |
+| | Throughput Benchmarking Test. |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Throughput |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To measure the throughput supported by the virtual Traffic |
+| | Classifier according to the RFC2544 methodology for a |
+| | user-defined set of vTC deployment configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: file: opnfv_yardstick_tc006.yaml |
+| | |
+| | packet_size: size of the packets to be used during the |
+| | throughput calculation. |
+| | Allowe values: [64, 128, 256, 512, 1024, 1280, 1518] |
+| | |
+| | vnic_type: type of VNIC to be used. |
+| | Allowed values are: |
+| | - normal: for default OvS port configuration |
+| | - direct: for SR-IOV port configuration |
+| | Default value: None |
+| | |
+| | vtc_flavor: OpenStack flavor to be used for the vTC |
+| | Default available values are: m1.small, m1.medium, |
+| | and m1.large, but the user can create his/her own |
+| | flavor and give it as input |
+| | Default value: None |
+| | |
+| | vlan_sender: vlan tag of the network on which the vTC will |
+| | receive traffic (VLAN Network 1). |
+| | Allowed values: range (1, 4096) |
+| | |
+| | vlan_receiver: vlan tag of the network on which the vTC |
+| | will send traffic back to the packet generator |
+| | (VLAN Network 2). |
+| | Allowed values: range (1, 4096) |
+| | |
+| | default_net_name: neutron name of the defaul network that |
+| | is used for access to the internet from the vTC |
+| | (vNIC 1). |
+| | |
+| | default_subnet_name: subnet name for vNIC1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_net_1_name: Neutron Name for VLAN Network 1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_subnet_1_name: Subnet Neutron name for VLAN Network 1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_net_2_name: Neutron Name for VLAN Network 2 |
+| | (information available through Neutron). |
+| | |
+| | vlan_subnet_2_name: Subnet Neutron name for VLAN Network 2 |
+| | (information available through Neutron). |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | DPDK pktgen |
+| | |
+| | DPDK Pktgen is not part of a Linux distribution, |
+| | hence it needs to be installed by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | DPDK Pktgen: DPDKpktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
+| | RFC 2544: rfc2544_ |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different flavors, vNIC type |
+| | and packet sizes. Default values exist as specified above. |
+| | The vNIC type and flavor MUST be specified by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The vTC has been successfully instantiated and configured. |
+| | The user has correctly assigned the values to the deployment |
+| | configuration parameters. |
+| | |
+| | - Multicast traffic MUST be enabled on the network. |
+| | The Data network switches need to be configured in |
+| | order to manage multicast traffic. |
+| | - In the case of SR-IOV vNICs use, SR-IOV compatible NICs |
+| | must be used on the compute node. |
+| | - Yarsdtick needs to be installed on a host connected to the |
+| | data network and the host must have 2 DPDK-compatible |
+| | NICs. Proper configuration of DPDK and DPDK pktgen is |
+| | required before to run the test case. |
+| | (For further instructions please refer to the ApexLake |
+| | documentation). |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | Description and expected results |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The vTC is deployed, according to the user-defined |
+| | configuration |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | The vTC is correctly deployed and configured as necessary |
+| | The initialization script has been correctly executed and |
+| | vTC is ready to receive and process the traffic. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | Test case is executed with the selected parameters: |
+| | - vTC flavor |
+| | - vNIC type |
+| | - packet size |
+| | The traffic is sent to the vTC using the maximum available |
+| | traffic rate for 60 seconds. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | The vTC instance forwards all the packets back to the packet |
+| | generator for 60 seconds, as specified by RFC 2544. |
+| | |
+| | Steps 3 and 4 are executed different times, with different |
+| | rates in order to find the maximum supported traffic rate |
+| | according to the current definition of throughput in RFC |
+| | 2544. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | The result of the test is a number between 0 and 100 which |
+| | represents the throughput in terms of percentage of the |
+| | available pktgen NIC bandwidth. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc007.rst b/docs/testing/user/userguide/opnfv_yardstick_tc007.rst
new file mode 100644
index 000000000..87663f816
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc007.rst
@@ -0,0 +1,162 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Intel Corporation and others.
+
+*************************************
+Yardstick Test Case Description TC007
+*************************************
+
+.. _DPDKpktgen: https://github.com/Pktgen/Pktgen-DPDK/
+.. _rfc2544: https://www.ietf.org/rfc/rfc2544.txt
+
++-----------------------------------------------------------------------------+
+|Network Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC007_Virtual Traffic Classifier Data Plane |
+| | Throughput Benchmarking Test in Presence of Noisy |
+| | neighbours |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Throughput |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To measure the throughput supported by the virtual Traffic |
+| | Classifier according to the RFC2544 methodology for a |
+| | user-defined set of vTC deployment configurations in the |
+| | presence of noisy neighbours. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc007.yaml |
+| | |
+| | packet_size: size of the packets to be used during the |
+| | throughput calculation. |
+| | Allowe values: [64, 128, 256, 512, 1024, 1280, 1518] |
+| | |
+| | vnic_type: type of VNIC to be used. |
+| | Allowed values are: |
+| | - normal: for default OvS port configuration |
+| | - direct: for SR-IOV port configuration |
+| | |
+| | vtc_flavor: OpenStack flavor to be used for the vTC |
+| | Default available values are: m1.small, m1.medium, |
+| | and m1.large, but the user can create his/her own |
+| | flavor and give it as input |
+| | |
+| | num_of_neighbours: Number of noisy neighbours (VMs) to be |
+| | instantiated during the experiment. |
+| | Allowed values: range (1, 10) |
+| | |
+| | amount_of_ram: RAM to be used by each neighbor. |
+| | Allowed values: ['250M', '1G', '2G', '3G', '4G', '5G', |
+| | '6G', '7G', '8G', '9G', '10G'] |
+| | Deault value: 256M |
+| | |
+| | number_of_cores: Number of noisy neighbours (VMs) to be |
+| | instantiated during the experiment. |
+| | Allowed values: range (1, 10) |
+| | Default value: 1 |
+| | |
+| | vlan_sender: vlan tag of the network on which the vTC will |
+| | receive traffic (VLAN Network 1). |
+| | Allowed values: range (1, 4096) |
+| | |
+| | vlan_receiver: vlan tag of the network on which the vTC |
+| | will send traffic back to the packet generator |
+| | (VLAN Network 2). |
+| | Allowed values: range (1, 4096) |
+| | |
+| | default_net_name: neutron name of the defaul network that |
+| | is used for access to the internet from the vTC |
+| | (vNIC 1). |
+| | |
+| | default_subnet_name: subnet name for vNIC1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_net_1_name: Neutron Name for VLAN Network 1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_subnet_1_name: Subnet Neutron name for VLAN Network 1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_net_2_name: Neutron Name for VLAN Network 2 |
+| | (information available through Neutron). |
+| | |
+| | vlan_subnet_2_name: Subnet Neutron name for VLAN Network 2 |
+| | (information available through Neutron). |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | DPDK pktgen |
+| | |
+| | DPDK Pktgen is not part of a Linux distribution, |
+| | hence it needs to be installed by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | DPDKpktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
+| | rfc2544_ |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different flavors, vNIC type |
+| | and packet sizes. Default values exist as specified above. |
+| | The vNIC type and flavor MUST be specified by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The vTC has been successfully instantiated and configured. |
+| | The user has correctly assigned the values to the deployment |
+| | configuration parameters. |
+| | |
+| | - Multicast traffic MUST be enabled on the network. |
+| | The Data network switches need to be configured in |
+| | order to manage multicast traffic. |
+| | - In the case of SR-IOV vNICs use, SR-IOV compatible NICs |
+| | must be used on the compute node. |
+| | - Yarsdtick needs to be installed on a host connected to the |
+| | data network and the host must have 2 DPDK-compatible |
+| | NICs. Proper configuration of DPDK and DPDK pktgen is |
+| | required before to run the test case. |
+| | (For further instructions please refer to the ApexLake |
+| | documentation). |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | Description and expected results |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The noisy neighbours are deployed as required by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | The vTC is deployed, according to the configuration required |
+| | by the user |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | The vTC is correctly deployed and configured as necessary. |
+| | The initialization script has been correctly executed and |
+| | the vTC is ready to receive and process the traffic. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | Test case is executed with the parameters specified by the |
+| | user: |
+| | - vTC flavor |
+| | - vNIC type |
+| | - packet size |
+| | The traffic is sent to the vTC using the maximum available |
+| | traffic rate |
+| | |
++--------------+--------------------------------------------------------------+
+|step 5 | The vTC instance forwards all the packets back to the |
+| | packet generator for 60 seconds, as specified by RFC 2544. |
+| | |
+| | Steps 4 and 5 are executed different times with different |
+| | with different traffic rates, in order to find the maximum |
+| | supported traffic rate, accoring to the current definition |
+| | of throughput in RFC 2544. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | The result of the test is a number between 0 and 100 which |
+| | represents the throughput in terms of percentage of the |
+| | available pktgen NIC bandwidth. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc008.rst b/docs/testing/user/userguide/opnfv_yardstick_tc008.rst
new file mode 100644
index 000000000..a4ecaf6ae
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc008.rst
@@ -0,0 +1,90 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC008
+*************************************
+
+.. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+
++-----------------------------------------------------------------------------+
+|Packet Loss Extended Test |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC008_NW PERF, Packet loss Extended Test |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of flows, packet size and throughput |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS network performance with regards to |
+| | flows and throughput, such as if and how different amounts |
+| | of packet sizes and flows matter for the throughput between |
+| | VMs on different compute blades. Typically e.g. the |
+| | performance of a vSwitch |
+| | depends on the number of flows running through it. Also |
+| | performance of other equipment or entities can depend |
+| | on the number of flows or the packet sizes used. |
+| | The purpose is also to be able to spot trends. Test results, |
+| | graphs ans similar shall be stored for comparison reasons and|
+| | product evolution understanding between different OPNFV |
+| | versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc008.yaml |
+| | |
+| | Packet size: 64, 128, 256, 512, 1024, 1280 and 1518 bytes. |
+| | |
+| | Number of ports: 1, 10, 50, 100, 500 and 1000. The amount of |
+| | configured ports map from 2 up to 1001000 flows, |
+| | respectively. Each packet_size/port_amount combination is run|
+| | ten times, for 20 seconds each. Then the next |
+| | packet_size/port_amount combination is run, and so on. |
+| | |
+| | The client and server are distributed on different HW. |
+| | |
+| | For SLA max_ppm is set to 1000. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | pktgen |
+| | |
+| | (Pktgen is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Docker |
+| | image. |
+| | As an example see the /yardstick/tools/ directory for how |
+| | to generate a Linux image with pktgen included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|references | pktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different packet sizes, amount |
+| | of flows and test duration. Default values exist. |
+| | |
+| | SLA (optional): max_ppm: The number of packets per million |
+| | packets sent that are acceptable to loose, not received. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with pktgen included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The hosts are installed, as server and client. pktgen is |
+| | invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc009.rst b/docs/testing/user/userguide/opnfv_yardstick_tc009.rst
new file mode 100644
index 000000000..d6f445361
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc009.rst
@@ -0,0 +1,89 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC009
+*************************************
+
+.. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+
++-----------------------------------------------------------------------------+
+|Packet Loss |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC009_NW PERF, Packet loss |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of flows, packets lost and throughput |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS network performance with regards to |
+| | flows and throughput, such as if and how different amounts |
+| | of flows matter for the throughput between VMs on different |
+| | compute blades. |
+| | Typically e.g. the performance of a vSwitch |
+| | depends on the number of flows running through it. Also |
+| | performance of other equipment or entities can depend |
+| | on the number of flows or the packet sizes used. |
+| | The purpose is also to be able to spot trends. Test results, |
+| | graphs ans similar shall be stored for comparison reasons and|
+| | product evolution understanding between different OPNFV |
+| | versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc009.yaml |
+| | |
+| | Packet size: 64 bytes |
+| | |
+| | Number of ports: 1, 10, 50, 100, 500 and 1000. The amount of |
+| | configured ports map from 2 up to 1001000 flows, |
+| | respectively. Each port amount is run ten times, for 20 |
+| | seconds each. Then the next port_amount is run, and so on. |
+| | |
+| | The client and server are distributed on different HW. |
+| | |
+| | For SLA max_ppm is set to 1000. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | pktgen |
+| | |
+| | (Pktgen is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Docker |
+| | image. |
+| | As an example see the /yardstick/tools/ directory for how |
+| | to generate a Linux image with pktgen included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|references | pktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different packet sizes, amount |
+| | of flows and test duration. Default values exist. |
+| | |
+| | SLA (optional): max_ppm: The number of packets per million |
+| | packets sent that are acceptable to loose, not received. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with pktgen included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The hosts are installed, as server and client. pktgen is |
+| | invoked and logs are produced and stored. |
+| | |
+| | Result: logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc010.rst b/docs/testing/user/userguide/opnfv_yardstick_tc010.rst
new file mode 100644
index 000000000..202307de6
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc010.rst
@@ -0,0 +1,154 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC010
+*************************************
+
+.. _lat_mem_rd: http://manpages.ubuntu.com/manpages/trusty/lat_mem_rd.8.html
+
++-----------------------------------------------------------------------------+
+|Memory Latency |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC010_MEMORY LATENCY |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Memory read latency (nanoseconds) |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC010 is to evaluate the IaaS compute |
+| | performance with regards to memory read latency. |
+| | It measures the memory read latency for varying memory sizes |
+| | and strides. Whole memory hierarchy is measured. |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Lmbench |
+| | |
+| | Lmbench is a suite of operating system microbenchmarks. This |
+| | test uses lat_mem_rd tool from that suite including: |
+| | * Context switching |
+| | * Networking: connection establishment, pipe, TCP, UDP, and |
+| | RPC hot potato |
+| | * File system creates and deletes |
+| | * Process creation |
+| | * Signal handling |
+| | * System call overhead |
+| | * Memory read latency |
+| | |
+| | (LMbench is not always part of a Linux distribution, hence |
+| | it needs to be installed. As an example see the |
+| | /yardstick/tools/ directory for how to generate a Linux |
+| | image with LMbench included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|test | LMbench lat_mem_rd benchmark measures memory read latency |
+|description | for varying memory sizes and strides. |
+| | |
+| | The benchmark runs as two nested loops. The outer loop is |
+| | the stride size. The inner loop is the array size. For each |
+| | array size, the benchmark creates a ring of pointers that |
+| | point backward one stride.Traversing the array is done by: |
+| | |
+| | p = (char **)*p; |
+| | |
+| | in a for loop (the over head of the for loop is not |
+| | significant; the loop is an unrolled loop 100 loads long). |
+| | The size of the array varies from 512 bytes to (typically) |
+| | eight megabytes. For the small sizes, the cache will have an |
+| | effect, and the loads will be much faster. This becomes much |
+| | more apparent when the data is plotted. |
+| | |
+| | Only data accesses are measured; the instruction cache is |
+| | not measured. |
+| | |
+| | The results are reported in nanoseconds per load and have |
+| | been verified accurate to within a few nanoseconds on an SGI |
+| | Indy. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | File: opnfv_yardstick_tc010.yaml |
+| | |
+| | * SLA (max_latency): 30 nanoseconds |
+| | * Stride - 128 bytes |
+| | * Stop size - 64 megabytes |
+| | * Iterations: 10 - test is run 10 times iteratively. |
+| | * Interval: 1 - there is 1 second delay between each |
+| | iteration. |
+| | |
+| | SLA is optional. The SLA in this test case serves as an |
+| | example. Considerably lower read latency is expected. |
+| | However, to cover most configurations, both baremetal and |
+| | fully virtualized ones, this value should be possible to |
+| | achieve and acceptable for black box testing. |
+| | Many heavy IO applications start to suffer badly if the |
+| | read latency is higher than this. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * strides; |
+| | * stop_size; |
+| | * iterations and intervals. |
+| | |
+| | Default values exist. |
+| | |
+| | SLA (optional) : max_latency: The maximum memory latency |
+| | that is accepted. |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is one of Yardstick's generic test. Thus it |
+| | is runnable on most of the scenarios. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | LMbench lat_mem_rd_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with Lmbench included in the image. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The host is installed as client. LMbench's lat_mem_rd tool |
+| | is invoked and logs are produced and stored. |
+| | |
+| | Result: logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | A host VM with LMbench installed is booted. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the host VM by using ssh. |
+| | 'lmbench_latency_benchmark' bash script is copyied from Jump |
+| | Host to the host VM via the ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | 'lmbench_latency_benchmark' script is invoked. LMbench's |
+| | lat_mem_rd benchmark starts to measures memory read latency |
+| | for varying memory sizes and strides. Memory read latency |
+| | are recorded and checked against the SLA. Logs are produced |
+| | and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | The host VM is deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Test fails if the measured memory latency is above the SLA |
+| | value or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc011.rst b/docs/testing/user/userguide/opnfv_yardstick_tc011.rst
new file mode 100644
index 000000000..48bdef497
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc011.rst
@@ -0,0 +1,123 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC011
+*************************************
+
+.. _iperf3: https://iperf.fr/
+
++-----------------------------------------------------------------------------+
+|Packet delay variation between VMs |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC011_PACKET DELAY VARIATION BETWEEN VMs |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | jitter: packet delay variation (ms) |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC011 is to evaluate the IaaS network |
+| | performance with regards to network jitter (packet delay |
+| | variation). |
+| | It measures the packet delay variation sending the packets |
+| | from one VM to the other. |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | iperf3 |
+| | |
+| | iPerf3 is a tool for active measurements of the maximum |
+| | achievable bandwidth on IP networks. It supports tuning of |
+| | various parameters related to timing, buffers and protocols. |
+| | The UDP protocols can be used to measure jitter delay. |
+| | |
+| | (iperf3 is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Docker |
+| | image. As an example see the /yardstick/tools/ directory for |
+| | how to generate a Linux image with pktgen included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|test | iperf3 test is invoked between a host VM and a target VM. |
+|description | |
+| | Jitter calculations are continuously computed by the server, |
+| | as specified by RTP in RFC 1889. The client records a 64 bit |
+| | second/microsecond timestamp in the packet. The server |
+| | computes the relative transit time as (server's receive time |
+| | - client's send time). The client's and server's clocks do |
+| | not need to be synchronized; any difference is subtracted |
+| | outin the jitter calculation. Jitter is the smoothed mean of |
+| | differences between consecutive transit times. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | File: opnfv_yardstick_tc011.yaml |
+| | |
+| | * options: |
+| | protocol: udp # The protocol used by iperf3 tools |
+| | bandwidth: 20m # It will send the given number of packets |
+| | without pausing |
+| | * runner: |
+| | duration: 30 # Total test duration 30 seconds. |
+| | |
+| | * SLA (optional): |
+| | jitter: 10 (ms) # The maximum amount of jitter that is |
+| | accepted. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * bandwidth: Test case can be configured with different |
+| | bandwidth. |
+| | |
+| | * duration: The test duration can be configured. |
+| | |
+| | * jitter: SLA is optional. The SLA in this test case |
+| | serves as an example. |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is one of Yardstick's generic test. Thus it |
+| | is runnable on most of the scenarios. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | iperf3_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with iperf3 included in the image. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | Two host VMs with iperf3 installed are booted, as server and |
+| | client. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the host VM by using ssh. |
+| | A iperf3 server is started on the server VM via the ssh |
+| | tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | iperf3 benchmark is invoked. Jitter is calculated and check |
+| | against the SLA. Logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | The host VMs are deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Test should not PASS if any jitter is above the optional SLA |
+| | value, or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc012.rst b/docs/testing/user/userguide/opnfv_yardstick_tc012.rst
new file mode 100644
index 000000000..b56e829f5
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc012.rst
@@ -0,0 +1,135 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC012
+*************************************
+
+.. _bw_mem: http://manpages.ubuntu.com/manpages/trusty/bw_mem.8.html
+
++-----------------------------------------------------------------------------+
+|Memory Bandwidth |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC012_MEMORY BANDWIDTH |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Memory read/write bandwidth (MBps) |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC012 is to evaluate the IaaS compute |
+| | performance with regards to memory throughput. |
+| | It measures the rate at which data can be read from and |
+| | written to the memory (this includes all levels of memory). |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | LMbench |
+| | |
+| | LMbench is a suite of operating system microbenchmarks. |
+| | This test uses bw_mem tool from that suite including: |
+| | * Cached file read |
+| | * Memory copy (bcopy) |
+| | * Memory read |
+| | * Memory write |
+| | * Pipe |
+| | * TCP |
+| | |
+| | (LMbench is not always part of a Linux distribution, hence |
+| | it needs to be installed. As an example see the |
+| | /yardstick/tools/ directory for how to generate a Linux |
+| | image with LMbench included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|test | LMbench bw_mem benchmark allocates twice the specified |
+|description | amount of memory, zeros it, and then times the copying of |
+| | the first half to the second half. The benchmark is invoked |
+| | in a host VM on a compute blade. Results are reported in |
+| | megabytes moved per second. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | File: opnfv_yardstick_tc012.yaml |
+| | |
+| | * SLA (optional): 15000 (MBps) min_bw: The minimum amount of |
+| | memory bandwidth that is accepted. |
+| | * Size: 10 240 kB - test allocates twice that size |
+| | (20 480kB) zeros it and then measures the time it takes to |
+| | copy from one side to another. |
+| | * Benchmark: rdwr - measures the time to read data into |
+| | memory and then write data to the same location. |
+| | * Warmup: 0 - the number of iterations to perform before |
+| | taking actual measurements. |
+| | * Iterations: 10 - test is run 10 times iteratively. |
+| | * Interval: 1 - there is 1 second delay between each |
+| | iteration. |
+| | |
+| | SLA is optional. The SLA in this test case serves as an |
+| | example. Considerably higher bandwidth is expected. |
+| | However, to cover most configurations, both baremetal and |
+| | fully virtualized ones, this value should be possible to |
+| | achieve and acceptable for black box testing. |
+| | Many heavy IO applications start to suffer badly if the |
+| | read/write bandwidths are lower than this. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * memory sizes; |
+| | * memory operations (such as rd, wr, rdwr, cp, frd, fwr, |
+| | fcp, bzero, bcopy); |
+| | * number of warmup iterations; |
+| | * iterations and intervals. |
+| | |
+| | Default values exist. |
+| | |
+| | SLA (optional) : min_bandwidth: The minimun memory bandwidth |
+| | that is accepted. |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is one of Yardstick's generic test. Thus it |
+| | is runnable on most of the scenarios. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | LMbench bw_mem_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with Lmbench included in the image. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | A host VM with LMbench installed is booted. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the host VM by using ssh. |
+| | "lmbench_bandwidth_benchmark" bash script is copied from |
+| | Jump Host to the host VM via ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | 'lmbench_bandwidth_benchmark' script is invoked. LMbench's |
+| | bw_mem benchmark starts to measures memory read/write |
+| | bandwidth. Memory read/write bandwidth results are recorded |
+| | and checked against the SLA. Logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | The host VM is deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Test fails if the measured memory bandwidth is below the SLA |
+| | value or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc014.rst b/docs/testing/user/userguide/opnfv_yardstick_tc014.rst
new file mode 100644
index 000000000..1b0d7831a
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc014.rst
@@ -0,0 +1,126 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC014
+*************************************
+
+.. _unixbench: https://github.com/kdlucas/byte-unixbench/blob/master/UnixBench
+
++-----------------------------------------------------------------------------+
+|Processing speed |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC014_PROCESSING SPEED |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | score of single cpu running, |
+| | score of parallel running |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC014 is to evaluate the IaaS compute |
+| | performance with regards to CPU processing speed. |
+| | It measures score of single cpu running and parallel |
+| | running. |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | UnixBench |
+| | |
+| | Unixbench is the most used CPU benchmarking software tool. |
+| | It can measure the performance of bash scripts, CPUs in |
+| | multithreading and single threading. It can also measure the |
+| | performance for parallel taks. Also, specific disk IO for |
+| | small and large files are performed. You can use it to |
+| | measure either linux dedicated servers and linux vps |
+| | servers, running CentOS, Debian, Ubuntu, Fedora and other |
+| | distros. |
+| | |
+| | (UnixBench is not always part of a Linux distribution, hence |
+| | it needs to be installed. As an example see the |
+| | /yardstick/tools/ directory for how to generate a Linux |
+| | image with UnixBench included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|test | The UnixBench runs system benchmarks in a host VM on a |
+|description | compute blade, getting information on the CPUs in the |
+| | system. If the system has more than one CPU, the tests will |
+| | be run twice -- once with a single copy of each test running |
+| | at once, and once with N copies, where N is the number of |
+| | CPUs. |
+| | |
+| | UnixBench will processs a set of results from a single test |
+| | by averaging the individal pass results into a single final |
+| | value. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc014.yaml |
+| | |
+| | run_mode: Run unixbench in quiet mode or verbose mode |
+| | test_type: dhry2reg, whetstone and so on |
+| | |
+| | For SLA with single_score and parallel_score, both can be |
+| | set by user, default is NA. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * test types; |
+| | * dhry2reg; |
+| | * whetstone. |
+| | |
+| | Default values exist. |
+| | |
+| | SLA (optional) : min_score: The minimun UnixBench score that |
+| | is accepted. |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is one of Yardstick's generic test. Thus it |
+| | is runnable on most of the scenarios. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | unixbench_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with unixbench included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | A host VM with UnixBench installed is booted. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the host VM by using ssh. |
+| | "unixbench_benchmark" bash script is copied from Jump Host |
+| | to the host VM via ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | UnixBench is invoked. All the tests are executed using the |
+| | "Run" script in the top-level of UnixBench directory. |
+| | The "Run" script will run a standard "index" test, and save |
+| | the report in the "results" directory. Then the report is |
+| | processed by "unixbench_benchmark" and checked againsted the |
+| | SLA. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | The host VM is deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc019.rst b/docs/testing/user/userguide/opnfv_yardstick_tc019.rst
new file mode 100644
index 000000000..1af502253
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc019.rst
@@ -0,0 +1,134 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC019
+*************************************
+
++-----------------------------------------------------------------------------+
+|Control Node Openstack Service High Availability |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC019_HA: Control node Openstack service down|
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of the |
+| | service provided by OpenStack (like nova-api, neutro-server) |
+| | on control node. |
+| | |
++--------------+--------------------------------------------------------------+
+|test method | This test case kills the processes of a specific Openstack |
+| | service on a selected control node, then checks whether the |
+| | request of the related Openstack command is OK and the killed|
+| | processes are recovered. |
+| | |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "kill-process" is |
+| | needed. This attacker includes three parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "kill-process" in this |
+| | test case. |
+| | 2) process_name: which is the process name of the specified |
+| | OpenStack service. If there are multiple processes use the |
+| | same name on the host, all of them are killed by this |
+| | attacker. |
+| | 3) host: which is the name of a control node being attacked. |
+| | |
+| | e.g. |
+| | -fault_type: "kill-process" |
+| | -process_name: "nova-api" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request |
+| | |
+| | 2. the "process" monitor check whether a process is running |
+| | on a specific node, which needs three parameters: |
+| | 1) monitor_type: which used for finding the monitor class and|
+| | related scritps. It should be always set to "process" |
+| | for this monitor. |
+| | 2) process_name: which is the process name for monitor |
+| | 3) host: which is the name of the node runing the process |
+| | |
+| | e.g. |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "nova image-list" |
+| | monitor2: |
+| | -monitor_type: "process" |
+| | -process_name: "nova-api" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there are two metrics: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
+| | 2)process_recover_time: which indicates the maximun time |
+| | (seconds) from the process being killed to recovered |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc019.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the kill process script with param value specified by |
+| | "process_name" |
+| | |
+| | Result: Process will be killed. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It will check the|
+| | status of the specified process on the host, and restart the |
+| | process if it is not running for next test cases |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc020.rst b/docs/testing/user/userguide/opnfv_yardstick_tc020.rst
new file mode 100644
index 000000000..f2f1d408b
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc020.rst
@@ -0,0 +1,141 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Intel Corporation and others.
+
+*************************************
+Yardstick Test Case Description TC020
+*************************************
+
+.. _DPDKpktgen: https://github.com/Pktgen/Pktgen-DPDK/
+.. _rfc2544: https://www.ietf.org/rfc/rfc2544.txt
+
++-----------------------------------------------------------------------------+
+|Network Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC0020_Virtual Traffic Classifier |
+| | Instantiation Test |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Failure |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To verify that a newly instantiated vTC is 'alive' and |
+| | functional and its instantiation is correctly supported by |
+| | the infrastructure. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc020.yaml |
+| | |
+| | vnic_type: type of VNIC to be used. |
+| | Allowed values are: |
+| | - normal: for default OvS port configuration |
+| | - direct: for SR-IOV port configuration |
+| | Default value: None |
+| | |
+| | vtc_flavor: OpenStack flavor to be used for the vTC |
+| | Default available values are: m1.small, m1.medium, |
+| | and m1.large, but the user can create his/her own |
+| | flavor and give it as input |
+| | Default value: None |
+| | |
+| | vlan_sender: vlan tag of the network on which the vTC will |
+| | receive traffic (VLAN Network 1). |
+| | Allowed values: range (1, 4096) |
+| | |
+| | vlan_receiver: vlan tag of the network on which the vTC |
+| | will send traffic back to the packet generator |
+| | (VLAN Network 2). |
+| | Allowed values: range (1, 4096) |
+| | |
+| | default_net_name: neutron name of the defaul network that |
+| | is used for access to the internet from the vTC |
+| | (vNIC 1). |
+| | |
+| | default_subnet_name: subnet name for vNIC1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_net_1_name: Neutron Name for VLAN Network 1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_subnet_1_name: Subnet Neutron name for VLAN Network 1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_net_2_name: Neutron Name for VLAN Network 2 |
+| | (information available through Neutron). |
+| | |
+| | vlan_subnet_2_name: Subnet Neutron name for VLAN Network 2 |
+| | (information available through Neutron). |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | DPDK pktgen |
+| | |
+| | DPDK Pktgen is not part of a Linux distribution, |
+| | hence it needs to be installed by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | DPDKpktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
+| | rfc2544_ |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different flavors, vNIC type |
+| | and packet sizes. Default values exist as specified above. |
+| | The vNIC type and flavor MUST be specified by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The vTC has been successfully instantiated and configured. |
+| | The user has correctly assigned the values to the deployment |
+| | configuration parameters. |
+| | |
+| | - Multicast traffic MUST be enabled on the network. |
+| | The Data network switches need to be configured in |
+| | order to manage multicast traffic. |
+| | Installation and configuration of smcroute is required |
+| | before to run the test case. |
+| | (For further instructions please refer to the ApexLake |
+| | documentation). |
+| | - In the case of SR-IOV vNICs use, SR-IOV compatible NICs |
+| | must be used on the compute node. |
+| | - Yarsdtick needs to be installed on a host connected to the |
+| | data network and the host must have 2 DPDK-compatible |
+| | NICs. Proper configuration of DPDK and DPDK pktgen is |
+| | required before to run the test case. |
+| | (For further instructions please refer to the ApexLake |
+| | documentation). |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | Description and expected results |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The vTC is deployed, according to the configuration provided |
+| | by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | The vTC is correctly deployed and configured as necessary. |
+| | The initialization script has been correctly executed and |
+| | the vTC is ready to receive and process the traffic. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | Test case is executed with the parameters specified by the |
+| | the user: |
+| | - vTC flavor |
+| | - vNIC type |
+| | A constant rate traffic is sent to the vTC for 10 seconds. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | The vTC instance tags all the packets and sends them back to |
+| | the packet generator for 10 seconds. |
+| | |
+| | The framework checks that the packet generator receives |
+| | back all the packets with the correct tag from the vTC. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | The vTC is deemed to be successfully instantiated if all |
+| | packets are sent back with the right tag as requested, |
+| | else it is deemed DoA (Dead on arrival) |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc021.rst b/docs/testing/user/userguide/opnfv_yardstick_tc021.rst
new file mode 100644
index 000000000..c7adc870a
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc021.rst
@@ -0,0 +1,157 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Intel Corporation and others.
+
+*************************************
+Yardstick Test Case Description TC021
+*************************************
+
+.. _DPDKpktgen: https://github.com/Pktgen/Pktgen-DPDK/
+.. _rfc2544: https://www.ietf.org/rfc/rfc2544.txt
+
++-----------------------------------------------------------------------------+
+|Network Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC0021_Virtual Traffic Classifier |
+| | Instantiation Test in Presence of Noisy Neighbours |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Failure |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To verify that a newly instantiated vTC is 'alive' and |
+| | functional and its instantiation is correctly supported by |
+| | the infrastructure in the presence of noisy neighbours. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc021.yaml |
+| | |
+| | vnic_type: type of VNIC to be used. |
+| | Allowed values are: |
+| | - normal: for default OvS port configuration |
+| | - direct: for SR-IOV port configuration |
+| | Default value: None |
+| | |
+| | vtc_flavor: OpenStack flavor to be used for the vTC |
+| | Default available values are: m1.small, m1.medium, |
+| | and m1.large, but the user can create his/her own |
+| | flavor and give it as input |
+| | Default value: None |
+| | |
+| | num_of_neighbours: Number of noisy neighbours (VMs) to be |
+| | instantiated during the experiment. |
+| | Allowed values: range (1, 10) |
+| | |
+| | amount_of_ram: RAM to be used by each neighbor. |
+| | Allowed values: ['250M', '1G', '2G', '3G', '4G', '5G', |
+| | '6G', '7G', '8G', '9G', '10G'] |
+| | Deault value: 256M |
+| | |
+| | number_of_cores: Number of noisy neighbours (VMs) to be |
+| | instantiated during the experiment. |
+| | Allowed values: range (1, 10) |
+| | Default value: 1 |
+| | |
+| | vlan_sender: vlan tag of the network on which the vTC will |
+| | receive traffic (VLAN Network 1). |
+| | Allowed values: range (1, 4096) |
+| | |
+| | vlan_receiver: vlan tag of the network on which the vTC |
+| | will send traffic back to the packet generator |
+| | (VLAN Network 2). |
+| | Allowed values: range (1, 4096) |
+| | |
+| | default_net_name: neutron name of the defaul network that |
+| | is used for access to the internet from the vTC |
+| | (vNIC 1). |
+| | |
+| | default_subnet_name: subnet name for vNIC1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_net_1_name: Neutron Name for VLAN Network 1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_subnet_1_name: Subnet Neutron name for VLAN Network 1 |
+| | (information available through Neutron). |
+| | |
+| | vlan_net_2_name: Neutron Name for VLAN Network 2 |
+| | (information available through Neutron). |
+| | |
+| | vlan_subnet_2_name: Subnet Neutron name for VLAN Network 2 |
+| | (information available through Neutron). |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | DPDK pktgen |
+| | |
+| | DPDK Pktgen is not part of a Linux distribution, |
+| | hence it needs to be installed by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | DPDK Pktgen: DPDK Pktgen: DPDKpktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
+| | RFC 2544: rfc2544_ |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different flavors, vNIC type |
+| | and packet sizes. Default values exist as specified above. |
+| | The vNIC type and flavor MUST be specified by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The vTC has been successfully instantiated and configured. |
+| | The user has correctly assigned the values to the deployment |
+| | configuration parameters. |
+| | |
+| | - Multicast traffic MUST be enabled on the network. |
+| | The Data network switches need to be configured in |
+| | order to manage multicast traffic. |
+| | Installation and configuration of smcroute is required |
+| | before to run the test case. |
+| | (For further instructions please refer to the ApexLake |
+| | documentation). |
+| | - In the case of SR-IOV vNICs use, SR-IOV compatible NICs |
+| | must be used on the compute node. |
+| | - Yarsdtick needs to be installed on a host connected to the |
+| | data network and the host must have 2 DPDK-compatible |
+| | NICs. Proper configuration of DPDK and DPDK pktgen is |
+| | required before to run the test case. |
+| | (For further instructions please refer to the ApexLake |
+| | documentation). |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | Description and expected results |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The noisy neighbours are deployed as required by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | The vTC is deployed, according to the configuration provided |
+| | by the user. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | The vTC is correctly deployed and configured as necessary. |
+| | The initialization script has been correctly executed and |
+| | the vTC is ready to receive and process the traffic. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | Test case is executed with the selected parameters: |
+| | - vTC flavor |
+| | - vNIC type |
+| | A constant rate traffic is sent to the vTC for 10 seconds. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 5 | The vTC instance tags all the packets and sends them back to |
+| | the packet generator for 10 seconds. |
+| | |
+| | The framework checks if the packet generator receives back |
+| | all the packets with the correct tag from the vTC. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | The vTC is deemed to be successfully instantiated if all |
+| | packets are sent back with the right tag as requested, |
+| | else it is deemed DoA (Dead on arrival) |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc024.rst b/docs/testing/user/userguide/opnfv_yardstick_tc024.rst
new file mode 100644
index 000000000..8d15e8d2f
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc024.rst
@@ -0,0 +1,76 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC024
+*************************************
+
+.. _man-pages: http://manpages.ubuntu.com/manpages/trusty/man1/mpstat.1.html
+
++-----------------------------------------------------------------------------+
+| CPU Load |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC024_CPU Load |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | CPU load |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the CPU load performance of the IaaS. This test |
+| | case should be run in parallel to other Yardstick test cases |
+| | and not run as a stand-alone test case. |
+| | Average, minimum and maximun values are obtained. |
+| | The purpose is also to be able to spot trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: cpuload.yaml (in the 'samples' directory) |
+| | |
+| | * interval: 1 - repeat, pausing every 1 seconds in-between. |
+| | * count: 10 - display statistics 10 times, then exit. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | mpstat |
+| | |
+| | (mpstat is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Glance |
+| | image. However, if mpstat is not present the TC instead uses |
+| | /proc/stats as source to produce "mpstat" output. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | man-pages_ |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * interval; |
+| | * count; |
+| | * runner Iteration and intervals. |
+| | |
+| | There are default values for each above-mentioned option. |
+| | Run in background with other test cases. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with mpstat included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The host is installed. The related TC, or TCs, is |
+| | invoked and mpstat logs are produced and stored. |
+| | |
+| | Result: Stored logs |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. CPU load results are fetched and stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc025.rst b/docs/testing/user/userguide/opnfv_yardstick_tc025.rst
new file mode 100644
index 000000000..0e2e9a5f8
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc025.rst
@@ -0,0 +1,123 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC025
+*************************************
+
++-----------------------------------------------------------------------------+
+|OpenStack Controller Node abnormally shutdown High Availability |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC025_HA: OpenStack Controller Node |
+| | abnormally shutdown |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of |
+| | controller node. When one of the controller node abnormally |
+| | shutdown, the service provided by it should be OK. |
+| | |
++--------------+--------------------------------------------------------------+
+|test method | This test case shutdowns a specified controller node with |
+| | some fault injection tools, then checks whether all services |
+| | provided by the controller node are OK with some monitor |
+| | tools. |
+| | |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "host-shutdown" is |
+| | needed. This attacker includes two parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "host-shutdown" in |
+| | this test case. |
+| | 2) host: the name of a controller node being attacked. |
+| | |
+| | e.g. |
+| | -fault_type: "host-shutdown" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, one kind of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request |
+| | |
+| | There are four instance of the "openstack-cmd" monitor: |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -api_name: "nova image-list" |
+| | monitor2: |
+| | -monitor_type: "openstack-cmd" |
+| | -api_name: "neutron router-list" |
+| | monitor3: |
+| | -monitor_type: "openstack-cmd" |
+| | -api_name: "heat stack-list" |
+| | monitor4: |
+| | -monitor_type: "openstack-cmd" |
+| | -api_name: "cinder list" |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there is one metric: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc019.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | shutdown script on the host |
+| | |
+| | Result: The host will be shutdown. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: All monitor result will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It restarts the |
+| | specified controller node if it is not restarted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc027.rst b/docs/testing/user/userguide/opnfv_yardstick_tc027.rst
new file mode 100644
index 000000000..125fd59fa
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc027.rst
@@ -0,0 +1,95 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC027
+*************************************
+
+.. _ipv6: https://wiki.opnfv.org/ipv6_opnfv_project
+
++-----------------------------------------------------------------------------+
+|IPv6 connectivity between nodes on the tenant network |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC027_IPv6 connectivity |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | RTT, Round Trip Time |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To do a basic verification that IPv6 connectivity is within |
+| | acceptable boundaries when ipv6 packets travel between hosts |
+| | located on same or different compute blades. |
+| | The purpose is also to be able to spot trends. Test results, |
+| | graphs and similar shall be stored for comparison reasons and|
+| | product evolution understanding between different OPNFV |
+| | versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc027.yaml |
+| | |
+| | Packet size 56 bytes. |
+| | SLA RTT is set to maximum 30 ms. |
+| | ipv6 test case can be configured as three independent modules|
+| | (setup, run, teardown). if you only want to setup ipv6 |
+| | testing environment, do some tests as you want, "run_step" |
+| | of task yaml file should be configured as "setup". if you |
+| | want to setup and run ping6 testing automatically, "run_step"|
+| | should be configured as "setup, run". and if you have had a |
+| | environment which has been setup, you only wan to verify the |
+| | connectivity of ipv6 network, "run_step" should be "run". Of |
+| | course, default is that three modules run sequentially. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | ping6 |
+| | |
+| | Ping6 is normally part of Linux distribution, hence it |
+| | doesn't need to be installed. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ipv6_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test case can be configured with different run step |
+| | you can run setup, run benchmark, teardown independently |
+| | SLA is optional. The SLA in this test case serves as an |
+| | example. Considerably lower RTT is expected. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with ping6 included in it. |
+| | |
+| | For Brahmaputra, a compass_os_nosdn_ha deploy scenario is |
+| | need. more installer and more sdn deploy scenario will be |
+| | supported soon |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | To setup IPV6 testing environment: |
+| | 1. disable security group |
+| | 2. create (ipv6, ipv4) router, network and subnet |
+| | 3. create VRouter, VM1, VM2 |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | To run ping6 to verify IPV6 connectivity : |
+| | 1. ssh to VM1 |
+| | 2. Ping6 to ipv6 router from VM1 |
+| | 3. Get the result(RTT) and logs are stored |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | To teardown IPV6 testing environment |
+| | 1. delete VRouter, VM1, VM2 |
+| | 2. delete (ipv6, ipv4) router, network and subnet |
+| | 3. enable security group |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Test should not PASS if any RTT is above the optional SLA |
+| | value, or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc028.rst b/docs/testing/user/userguide/opnfv_yardstick_tc028.rst
new file mode 100644
index 000000000..24206f33f
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc028.rst
@@ -0,0 +1,70 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co., Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC028
+*************************************
+
+.. _Cyclictest: https://rt.wiki.kernel.org/index.php/Cyclictest
+
++-----------------------------------------------------------------------------+
+|KVM Latency measurements |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC028_KVM Latency measurements |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | min, avg and max latency |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS KVM virtualization capability with |
+| | regards to min, avg and max latency. |
+| | The purpose is also to be able to spot trends. Test results, |
+| | graphs and similar shall be stored for comparison reasons |
+| | and product evolution understanding between different OPNFV |
+| | versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: samples/cyclictest-node-context.yaml |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Cyclictest |
+| | |
+| | (Cyclictest is not always part of a Linux distribution, |
+| | hence it needs to be installed. As an example see the |
+| | /yardstick/tools/ directory for how to generate a Linux |
+| | image with cyclictest included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Cyclictest_ |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | This test case is mainly for kvm4nfv project CI verify. |
+| | Upgrade host linux kernel, boot a gust vm update it's linux |
+| | kernel, and then run the cyclictest to test the new kernel |
+| | is work well. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test kernel rpm, test sequence scripts and test guest |
+|conditions | image need put the right folders as specified in the test |
+| | case yaml file. |
+| | The test guest image needs with cyclictest included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The host and guest os kernel is upgraded. Cyclictest is |
+| | invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc037.rst b/docs/testing/user/userguide/opnfv_yardstick_tc037.rst
new file mode 100644
index 000000000..5a6e1eaae
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc037.rst
@@ -0,0 +1,167 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC037
+*************************************
+
+.. _cirros-image: https://download.cirros-cloud.net
+.. _Ping: https://linux.die.net/man/8/ping
+.. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+.. _mpstat: http://www.linuxcommand.org/man_pages/mpstat1.html
+
++-----------------------------------------------------------------------------+
+|Latency, CPU Load, Throughput, Packet Loss |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC037_LATENCY,CPU LOAD,THROUGHPUT, |
+| | PACKET LOSS |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of flows, latency, throughput, packet loss |
+| | CPU utilization percentage, CPU interrupt per second |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC037 is to evaluate the IaaS compute |
+| | capacity and network performance with regards to CPU |
+| | utilization, packet flows and network throughput, such as if |
+| | and how different amounts of flows matter for the throughput |
+| | between hosts on different compute blades, and the CPU load |
+| | variation. |
+| | |
+| | Typically e.g. the performance of a vSwitch depends on the |
+| | number of flows running through it. Also performance of |
+| | other equipment or entities can depend on the number of |
+| | flows or the packet sizes used |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Ping, Pktgen, mpstat |
+| | |
+| | Ping is a computer network administration software utility |
+| | used to test the reachability of a host on an Internet |
+| | Protocol (IP) network. It measures the round-trip time for |
+| | packet sent from the originating host to a destination |
+| | computer that are echoed back to the source. |
+| | |
+| | Linux packet generator is a tool to generate packets at very |
+| | high speed in the kernel. pktgen is mainly used to drive and |
+| | LAN equipment test network. pktgen supports multi threading. |
+| | To generate random MAC address, IP address, port number UDP |
+| | packets, pktgen uses multiple CPU processors in the |
+| | different PCI bus (PCI, PCIe bus) with Gigabit Ethernet |
+| | tested (pktgen performance depends on the CPU processing |
+| | speed, memory delay, PCI bus speed hardware parameters), |
+| | Transmit data rate can be even larger than 10GBit/s. Visible |
+| | can satisfy most card test requirements. |
+| | |
+| | The mpstat command writes to standard output activities for |
+| | each available processor, processor 0 being the first one. |
+| | Global average activities among all processors are also |
+| | reported. The mpstat command can be used both on SMP and UP |
+| | machines, but in the latter, only global average activities |
+| | will be printed. |
+| | |
+| | (Ping is normally part of any Linux distribution, hence it |
+| | doesn't need to be installed. It is also part of the |
+| | Yardstick Docker image. |
+| | For example also a Cirros image can be downloaded from |
+| | cirros-image_, it includes ping. |
+| | |
+| | Pktgen and mpstat are not always part of a Linux |
+| | distribution, hence it needs to be installed. It is part of |
+| | the Yardstick Docker image. |
+| | As an example see the /yardstick/tools/ directory for how |
+| | to generate a Linux image with pktgen and mpstat included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|test | This test case uses Pktgen to generate packet flow between |
+|description | two hosts for simulating network workloads on the SUT. |
+| | Ping packets (ICMP protocol's mandatory ECHO_REQUEST |
+| | datagram) are sent from a host VM to the target VM(s) to |
+| | elicit ICMP ECHO_RESPONSE, meanwhile CPU activities are |
+| | monitored by mpstat. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc037.yaml |
+| | |
+| | Packet size is set to 64 bytes. |
+| | Number of ports: 1, 10, 50, 100, 300, 500, 750 and 1000. |
+| | The amount configured ports map from 2 up to 1001000 flows, |
+| | respectively. Each port amount is run two times, for 20 |
+| | seconds each. Then the next port_amount is run, and so on. |
+| | During the test CPU load on both client and server, and the |
+| | network latency between the client and server are measured. |
+| | The client and server are distributed on different hardware. |
+| | mpstat monitoring interval is set to 1 second. |
+| | ping packet size is set to 100 bytes. |
+| | For SLA max_ppm is set to 1000. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * pktgen packet sizes; |
+| | * amount of flows; |
+| | * test duration; |
+| | * ping packet size; |
+| | * mpstat monitor interval. |
+| | |
+| | Default values exist. |
+| | |
+| | SLA (optional): max_ppm: The number of packets per million |
+| | packets sent that are acceptable to loose, not received. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Ping_ |
+| | |
+| | mpstat_ |
+| | |
+| | pktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with pktgen, mpstat included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | Two host VMs are booted, as server and client. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Yardstick is connected with the server VM by using ssh. |
+| | 'pktgen_benchmark', "ping_benchmark" bash script are copyied |
+| | from Jump Host to the server VM via the ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | An IP table is setup on server to monitor for received |
+| | packets. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | pktgen is invoked to generate packet flow between two server |
+| | and client for simulating network workloads on the SUT. Ping |
+| | is invoked. Ping packets are sent from server VM to client |
+| | VM. mpstat is invoked, recording activities for each |
+| | available processor. Results are processed and checked |
+| | against the SLA. Logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 5 | Two host VMs are deleted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc038.rst b/docs/testing/user/userguide/opnfv_yardstick_tc038.rst
new file mode 100644
index 000000000..692c76819
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc038.rst
@@ -0,0 +1,104 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TC038
+*************************************
+
+.. _cirros: https://download.cirros-cloud.net
+.. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+
++-----------------------------------------------------------------------------+
+|Latency, CPU Load, Throughput, Packet Loss (Extended measurements) |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC038_Latency,CPU Load,Throughput,Packet Loss|
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of flows, latency, throughput, CPU load, packet loss |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS network performance with regards to |
+| | flows and throughput, such as if and how different amounts |
+| | of flows matter for the throughput between hosts on different|
+| | compute blades. Typically e.g. the performance of a vSwitch |
+| | depends on the number of flows running through it. Also |
+| | performance of other equipment or entities can depend |
+| | on the number of flows or the packet sizes used. |
+| | The purpose is also to be able to spot trends. Test results, |
+| | graphs ans similar shall be stored for comparison reasons and|
+| | product evolution understanding between different OPNFV |
+| | versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc038.yaml |
+| | |
+| | Packet size: 64 bytes |
+| | Number of ports: 1, 10, 50, 100, 300, 500, 750 and 1000. |
+| | The amount configured ports map from 2 up to 1001000 flows, |
+| | respectively. Each port amount is run ten times, for 20 |
+| | seconds each. Then the next port_amount is run, and so on. |
+| | During the test CPU load on both client and server, and the |
+| | network latency between the client and server are measured. |
+| | The client and server are distributed on different HW. |
+| | For SLA max_ppm is set to 1000. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | pktgen |
+| | |
+| | (Pktgen is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Glance |
+| | image. |
+| | As an example see the /yardstick/tools/ directory for how |
+| | to generate a Linux image with pktgen included.) |
+| | |
+| | ping |
+| | |
+| | Ping is normally part of any Linux distribution, hence it |
+| | doesn't need to be installed. It is also part of the |
+| | Yardstick Glance image. |
+| | (For example also a cirros_ image can be downloaded, it |
+| | includes ping) |
+| | |
+| | mpstat |
+| | |
+| | (Mpstat is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Glance |
+| | image. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Ping and Mpstat man pages |
+| | |
+| | pktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different packet sizes, amount |
+| | of flows and test duration. Default values exist. |
+| | |
+| | SLA (optional): max_ppm: The number of packets per million |
+| | packets sent that are acceptable to loose, not received. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with pktgen included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The hosts are installed, as server and client. pktgen is |
+| | invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc040.rst b/docs/testing/user/userguide/opnfv_yardstick_tc040.rst
new file mode 100644
index 000000000..d62fbf787
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc040.rst
@@ -0,0 +1,65 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC040
+*************************************
+
+.. _Parser: https://wiki.opnfv.org/parser
+
++-----------------------------------------------------------------------------+
+|Verify Parser Yang-to-Tosca |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC040 Verify Parser Yang-to-Tosca |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | 1. tosca file which is converted from yang file by Parser |
+| | 2. result whether the output is same with expected outcome |
++--------------+--------------------------------------------------------------+
+|test purpose | To verify the function of Yang-to-Tosca in Parser. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc040.yaml |
+| | |
+| | yangfile: the path of the yangfile which you want to convert |
+| | toscafile: the path of the toscafile which is your expected |
+| | outcome. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Parser |
+| | |
+| | (Parser is not part of a Linux distribution, hence it |
+| | needs to be installed. As an example see the |
+| | /yardstick/benchmark/scenarios/parser/parser_setup.sh for |
+| | how to install it manual. Of course, it will be installed |
+| | and uninstalled automatically when you run this test case |
+| | by yardstick) |
++--------------+--------------------------------------------------------------+
+|references | Parser_ |
+| | |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different path of yangfile and |
+| | toscafile to fit your real environment to verify Parser |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | No POD specific requirements have been identified. |
+|conditions | it can be run without VM |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | parser is installed without VM, running Yang-to-Tosca module |
+| | to convert yang file to tosca file, validating output against|
+| | expected outcome. |
+| | |
+| | Result: Logs are stored. |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if output is different with expected outcome |
+| | or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc042.rst b/docs/testing/user/userguide/opnfv_yardstick_tc042.rst
new file mode 100644
index 000000000..8660d9297
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc042.rst
@@ -0,0 +1,87 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, ZTE and others.
+
+***************************************
+Yardstick Test Case Description TC0042
+***************************************
+
+.. _DPDK: http://dpdk.org/doc/guides/index.html
+.. _Testpmd: http://dpdk.org/doc/guides/testpmd_app_ug/index.html
+.. _Pktgen-dpdk: http://pktgen.readthedocs.io/en/latest/index.html
+
++-----------------------------------------------------------------------------+
+|Network Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC042_DPDK pktgen latency measurements |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | L2 Network Latency |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | Measure L2 network latency when DPDK is enabled between hosts|
+| | on different compute blades. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc042.yaml |
+| | |
+| | * Packet size: 64 bytes |
+| | * SLA(max_latency): 100usec |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | DPDK_ |
+| | Pktgen-dpdk_ |
+| | |
+| | (DPDK and Pktgen-dpdk are not part of a Linux distribution, |
+| | hence they needs to be installed. |
+| | As an example see the /yardstick/tools/ directory for how to |
+| | generate a Linux image with DPDK and pktgen-dpdk included.) |
+| | |
++--------------+--------------------------------------------------------------+
+|references | DPDK_ |
+| | |
+| | Pktgen-dpdk_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different packet sizes. Default |
+| | values exist. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with DPDK and pktgen-dpdk included in it. |
+| | |
+| | The NICs of compute nodes must support DPDK on POD. |
+| | |
+| | And at least compute nodes setup hugepage. |
+| | |
+| | If you want to achievement a hight performance result, it is |
+| | recommend to use NUAM, CPU pin, OVS and so on. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The hosts are installed on different blades, as server and |
+| | client. Both server and client have three interfaces. The |
+| | first one is management such as ssh. The other two are used |
+| | by DPDK. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Testpmd_ is invoked with configurations to forward packets |
+| | from one DPDK port to the other on server. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | Pktgen-dpdk is invoked with configurations as a traffic |
+| | generator and logs are produced and stored on client. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc043.rst b/docs/testing/user/userguide/opnfv_yardstick_tc043.rst
new file mode 100644
index 000000000..a873696dc
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc043.rst
@@ -0,0 +1,102 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC043
+*************************************
+
+.. _cirros-image: https://download.cirros-cloud.net
+.. _Ping: https://linux.die.net/man/8/ping
+
++-----------------------------------------------------------------------------+
+|Network Latency Between NFVI Nodes |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC043_LATENCY_BETWEEN_NFVI_NODES |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | RTT (Round Trip Time) |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC043 is to do a basic verification that |
+| | network latency is within acceptable boundaries when packets |
+| | travel between different NFVI nodes. |
+| | |
+| | The purpose is also to be able to spot the trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | ping |
+| | |
+| | Ping is a computer network administration software utility |
+| | used to test the reachability of a host on an Internet |
+| | Protocol (IP) network. It measures the round-trip time for |
+| | packet sent from the originating host to a destination |
+| | computer that are echoed back to the source. |
+| | |
++--------------+--------------------------------------------------------------+
+|test topology | Ping packets (ICMP protocol's mandatory ECHO_REQUEST |
+| | datagram) are sent from host node to target node to elicit |
+| | ICMP ECHO_RESPONSE. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc043.yaml |
+| | |
+| | Packet size 100 bytes. Total test duration 600 seconds. |
+| | One ping each 10 seconds. SLA RTT is set to maximum 10 ms. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | This test case can be configured with different: |
+| | |
+| | * packet sizes; |
+| | * burst sizes; |
+| | * ping intervals; |
+| | * test durations; |
+| | * test iterations. |
+| | |
+| | Default values exist. |
+| | |
+| | SLA is optional. The SLA in this test case serves as an |
+| | example. Considerably lower RTT is expected, and also normal |
+| | to achieve in balanced L2 environments. However, to cover |
+| | most configurations, both bare metal and fully virtualized |
+| | ones, this value should be possible to achieve and |
+| | acceptable for black box testing. Many real time |
+| | applications start to suffer badly if the RTT time is higher |
+| | than this. Some may suffer bad also close to this RTT, while |
+| | others may not suffer at all. It is a compromise that may |
+| | have to be tuned for different configuration purposes. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Ping_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre_test | Each pod node must have ping included in it. |
+|conditions | |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | Yardstick is connected with the NFVI node by using ssh. |
+| | 'ping_benchmark' bash script is copyied from Jump Host to |
+| | the NFVI node via the ssh tunnel. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Ping is invoked. Ping packets are sent from server node to |
+| | client node. RTT results are calculated and checked against |
+| | the SLA. Logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Test should not PASS if any RTT is above the optional SLA |
+| | value, or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc044.rst b/docs/testing/user/userguide/opnfv_yardstick_tc044.rst
new file mode 100644
index 000000000..2be8517a1
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc044.rst
@@ -0,0 +1,82 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC044
+*************************************
+
+.. _man-pages: http://manpages.ubuntu.com/manpages/trusty/en/man1/free.1.html
+
++-----------------------------------------------------------------------------+
+|Memory Utilization |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC044_Memory Utilization |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Memory utilization |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS compute capability with regards to |
+| | memory utilization.This test case should be run in parallel |
+| | to other Yardstick test cases and not run as a stand-alone |
+| | test case. |
+| | Measure the memory usage statistics including used memory, |
+| | free memory, buffer, cache and shared memory. |
+| | Both average and maximun values are obtained. |
+| | The purpose is also to be able to spot trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | File: memload.yaml (in the 'samples' directory) |
+| | |
+| | * interval: 1 - repeat, pausing every 1 seconds in-between. |
+| | * count: 10 - display statistics 10 times, then exit. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | free |
+| | |
+| | free provides information about unused and used memory and |
+| | swap space on any computer running Linux or another Unix-like|
+| | operating system. |
+| | free is normally part of a Linux distribution, hence it |
+| | doesn't needs to be installed. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | man-pages_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * interval; |
+| | * count; |
+| | * runner Iteration and intervals. |
+| | |
+| | There are default values for each above-mentioned option. |
+| | Run in background with other test cases. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with free included in the image. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The host is installed as client. The related TC, or TCs, is |
+| | invoked and free logs are produced and stored. |
+| | |
+| | Result: logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. Memory utilization results are fetched and stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc045.rst b/docs/testing/user/userguide/opnfv_yardstick_tc045.rst
new file mode 100644
index 000000000..0b0993c34
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc045.rst
@@ -0,0 +1,139 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC045
+*************************************
+
++-----------------------------------------------------------------------------+
+|Control Node Openstack Service High Availability - Neutron Server |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC045: Control node Openstack service down - |
+| | neutron server |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of the |
+| | network service provided by OpenStack (neutro-server) on |
+| | control node. |
+| | |
++--------------+--------------------------------------------------------------+
+|test method | This test case kills the processes of neutron-server service |
+| | on a selected control node, then checks whether the request |
+| | of the related Openstack command is OK and the killed |
+| | processes are recovered. |
+| | |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "kill-process" is |
+| | needed. This attacker includes three parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "kill-process" in this |
+| | test case. |
+| | 2) process_name: which is the process name of the specified |
+| | OpenStack service. If there are multiple processes use the |
+| | same name on the host, all of them are killed by this |
+| | attacker. |
+| | In this case. This parameter should always set to "neutron- |
+| | server". |
+| | 3) host: which is the name of a control node being attacked. |
+| | |
+| | e.g. |
+| | -fault_type: "kill-process" |
+| | -process_name: "neutron-server" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request. |
+| | In this case, the command name should be neutron related |
+| | commands. |
+| | |
+| | 2. the "process" monitor check whether a process is running |
+| | on a specific node, which needs three parameters: |
+| | 1) monitor_type: which used for finding the monitor class and|
+| | related scritps. It should be always set to "process" |
+| | for this monitor. |
+| | 2) process_name: which is the process name for monitor |
+| | 3) host: which is the name of the node runing the process |
+| | |
+| | e.g. |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "neutron agent-list" |
+| | monitor2: |
+| | -monitor_type: "process" |
+| | -process_name: "neutron-server" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there are two metrics: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
+| | 2)process_recover_time: which indicates the maximun time |
+| | (seconds) from the process being killed to recovered |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc045.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the kill process script with param value specified by |
+| | "process_name" |
+| | |
+| | Result: Process will be killed. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It will check the|
+| | status of the specified process on the host, and restart the |
+| | process if it is not running for next test cases |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc046.rst b/docs/testing/user/userguide/opnfv_yardstick_tc046.rst
new file mode 100644
index 000000000..cce6c6884
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc046.rst
@@ -0,0 +1,138 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC046
+*************************************
+
++-----------------------------------------------------------------------------+
+|Control Node Openstack Service High Availability - Keystone |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC046: Control node Openstack service down - |
+| | keystone |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of the |
+| | user service provided by OpenStack (keystone) on control |
+| | node. |
+| | |
++--------------+--------------------------------------------------------------+
+|test method | This test case kills the processes of keystone service on a |
+| | selected control node, then checks whether the request of |
+| | the related Openstack command is OK and the killed processes |
+| | are recovered. |
+| | |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "kill-process" is |
+| | needed. This attacker includes three parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "kill-process" in this |
+| | test case. |
+| | 2) process_name: which is the process name of the specified |
+| | OpenStack service. If there are multiple processes use the |
+| | same name on the host, all of them are killed by this |
+| | attacker. |
+| | In this case. This parameter should always set to "keystone" |
+| | 3) host: which is the name of a control node being attacked. |
+| | |
+| | e.g. |
+| | -fault_type: "kill-process" |
+| | -process_name: "keystone" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request. |
+| | In this case, the command name should be keystone related |
+| | commands. |
+| | |
+| | 2. the "process" monitor check whether a process is running |
+| | on a specific node, which needs three parameters: |
+| | 1) monitor_type: which used for finding the monitor class and|
+| | related scritps. It should be always set to "process" |
+| | for this monitor. |
+| | 2) process_name: which is the process name for monitor |
+| | 3) host: which is the name of the node runing the process |
+| | |
+| | e.g. |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "keystone user-list" |
+| | monitor2: |
+| | -monitor_type: "process" |
+| | -process_name: "keystone" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there are two metrics: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
+| | 2)process_recover_time: which indicates the maximun time |
+| | (seconds) from the process being killed to recovered |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc046.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the kill process script with param value specified by |
+| | "process_name" |
+| | |
+| | Result: Process will be killed. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It will check the|
+| | status of the specified process on the host, and restart the |
+| | process if it is not running for next test cases |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc047.rst b/docs/testing/user/userguide/opnfv_yardstick_tc047.rst
new file mode 100644
index 000000000..95158cfd6
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc047.rst
@@ -0,0 +1,139 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC047
+*************************************
+
++-----------------------------------------------------------------------------+
+|Control Node Openstack Service High Availability - Glance Api |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC047: Control node Openstack service down - |
+| | glance api |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of the |
+| | image service provided by OpenStack (glance-api) on control |
+| | node. |
+| | |
++--------------+--------------------------------------------------------------+
+|test method | This test case kills the processes of glance-api service on |
+| | a selected control node, then checks whether the request of |
+| | the related Openstack command is OK and the killed processes |
+| | are recovered. |
+| | |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "kill-process" is |
+| | needed. This attacker includes three parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "kill-process" in this |
+| | test case. |
+| | 2) process_name: which is the process name of the specified |
+| | OpenStack service. If there are multiple processes use the |
+| | same name on the host, all of them are killed by this |
+| | attacker. |
+| | In this case. This parameter should always set to "glance- |
+| | api". |
+| | 3) host: which is the name of a control node being attacked. |
+| | |
+| | e.g. |
+| | -fault_type: "kill-process" |
+| | -process_name: "glance-api" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request. |
+| | In this case, the command name should be glance related |
+| | commands. |
+| | |
+| | 2. the "process" monitor check whether a process is running |
+| | on a specific node, which needs three parameters: |
+| | 1) monitor_type: which used for finding the monitor class and|
+| | related scritps. It should be always set to "process" |
+| | for this monitor. |
+| | 2) process_name: which is the process name for monitor |
+| | 3) host: which is the name of the node runing the process |
+| | |
+| | e.g. |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "glance image-list" |
+| | monitor2: |
+| | -monitor_type: "process" |
+| | -process_name: "glance-api" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there are two metrics: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
+| | 2)process_recover_time: which indicates the maximun time |
+| | (seconds) from the process being killed to recovered |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc047.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the kill process script with param value specified by |
+| | "process_name" |
+| | |
+| | Result: Process will be killed. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It will check the|
+| | status of the specified process on the host, and restart the |
+| | process if it is not running for next test cases |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc048.rst b/docs/testing/user/userguide/opnfv_yardstick_tc048.rst
new file mode 100644
index 000000000..21c00d1fe
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc048.rst
@@ -0,0 +1,139 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC048
+*************************************
+
++-----------------------------------------------------------------------------+
+|Control Node Openstack Service High Availability - Cinder Api |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC048: Control node Openstack service down - |
+| | cinder api |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of the |
+| | volume service provided by OpenStack (cinder-api) on control |
+| | node. |
+| | |
++--------------+--------------------------------------------------------------+
+|test method | This test case kills the processes of cinder-api service on |
+| | a selected control node, then checks whether the request of |
+| | the related Openstack command is OK and the killed processes |
+| | are recovered. |
+| | |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "kill-process" is |
+| | needed. This attacker includes three parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "kill-process" in this |
+| | test case. |
+| | 2) process_name: which is the process name of the specified |
+| | OpenStack service. If there are multiple processes use the |
+| | same name on the host, all of them are killed by this |
+| | attacker. |
+| | In this case. This parameter should always set to "cinder- |
+| | api". |
+| | 3) host: which is the name of a control node being attacked. |
+| | |
+| | e.g. |
+| | -fault_type: "kill-process" |
+| | -process_name: "cinder-api" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request. |
+| | In this case, the command name should be cinder related |
+| | commands. |
+| | |
+| | 2. the "process" monitor check whether a process is running |
+| | on a specific node, which needs three parameters: |
+| | 1) monitor_type: which used for finding the monitor class and|
+| | related scritps. It should be always set to "process" |
+| | for this monitor. |
+| | 2) process_name: which is the process name for monitor |
+| | 3) host: which is the name of the node runing the process |
+| | |
+| | e.g. |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "cinder list" |
+| | monitor2: |
+| | -monitor_type: "process" |
+| | -process_name: "cinder-api" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there are two metrics: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
+| | 2)process_recover_time: which indicates the maximun time |
+| | (seconds) from the process being killed to recovered |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc048.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the kill process script with param value specified by |
+| | "process_name" |
+| | |
+| | Result: Process will be killed. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It will check the|
+| | status of the specified process on the host, and restart the |
+| | process if it is not running for next test cases |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc049.rst b/docs/testing/user/userguide/opnfv_yardstick_tc049.rst
new file mode 100644
index 000000000..f58bb9989
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc049.rst
@@ -0,0 +1,139 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC049
+*************************************
+
++-----------------------------------------------------------------------------+
+|Control Node Openstack Service High Availability - Swift Proxy |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC049: Control node Openstack service down - |
+| | swift proxy |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of the |
+| | storage service provided by OpenStack (swift-proxy) on |
+| | control node. |
+| | |
++--------------+--------------------------------------------------------------+
+|test method | This test case kills the processes of swift-proxy service on |
+| | a selected control node, then checks whether the request of |
+| | the related Openstack command is OK and the killed processes |
+| | are recovered. |
+| | |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "kill-process" is |
+| | needed. This attacker includes three parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "kill-process" in this |
+| | test case. |
+| | 2) process_name: which is the process name of the specified |
+| | OpenStack service. If there are multiple processes use the |
+| | same name on the host, all of them are killed by this |
+| | attacker. |
+| | In this case. This parameter should always set to "swift- |
+| | proxy". |
+| | 3) host: which is the name of a control node being attacked. |
+| | |
+| | e.g. |
+| | -fault_type: "kill-process" |
+| | -process_name: "swift-proxy" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request. |
+| | In this case, the command name should be swift related |
+| | commands. |
+| | |
+| | 2. the "process" monitor check whether a process is running |
+| | on a specific node, which needs three parameters: |
+| | 1) monitor_type: which used for finding the monitor class and|
+| | related scritps. It should be always set to "process" |
+| | for this monitor. |
+| | 2) process_name: which is the process name for monitor |
+| | 3) host: which is the name of the node runing the process |
+| | |
+| | e.g. |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "swift stat" |
+| | monitor2: |
+| | -monitor_type: "process" |
+| | -process_name: "swift-proxy" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there are two metrics: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
+| | 2)process_recover_time: which indicates the maximun time |
+| | (seconds) from the process being killed to recovered |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc049.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the kill process script with param value specified by |
+| | "process_name" |
+| | |
+| | Result: Process will be killed. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It will check the|
+| | status of the specified process on the host, and restart the |
+| | process if it is not running for next test cases |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc050.rst b/docs/testing/user/userguide/opnfv_yardstick_tc050.rst
new file mode 100644
index 000000000..8890c9d53
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc050.rst
@@ -0,0 +1,135 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC050
+*************************************
+
++-----------------------------------------------------------------------------+
+|OpenStack Controller Node Network High Availability |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC050: OpenStack Controller Node Network |
+| | High Availability |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of control |
+| | node. When one of the controller failed to connect the |
+| | network, which breaks down the Openstack services on this |
+| | node. These Openstack service should able to be accessed by |
+| | other controller nodes, and the services on failed |
+| | controller node should be isolated. |
++--------------+--------------------------------------------------------------+
+|test method | This test case turns off the network interfaces of a |
+| | specified control node, then checks whether all services |
+| | provided by the control node are OK with some monitor tools. |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "close-interface" is |
+| | needed. This attacker includes three parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "close-interface" in |
+| | this test case. |
+| | 2) host: which is the name of a control node being attacked. |
+| | 3) interface: the network interface to be turned off. |
+| | |
+| | There are four instance of the "close-interface" monitor: |
+| | attacker1(for public netork): |
+| | -fault_type: "close-interface" |
+| | -host: node1 |
+| | -interface: "br-ex" |
+| | attacker2(for management netork): |
+| | -fault_type: "close-interface" |
+| | -host: node1 |
+| | -interface: "br-mgmt" |
+| | attacker3(for storage netork): |
+| | -fault_type: "close-interface" |
+| | -host: node1 |
+| | -interface: "br-storage" |
+| | attacker4(for private netork): |
+| | -fault_type: "close-interface" |
+| | -host: node1 |
+| | -interface: "br-mesh" |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, the monitor named "openstack-cmd" is |
+| | needed. The monitor needs needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request |
+| | |
+| | There are four instance of the "openstack-cmd" monitor: |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "nova image-list" |
+| | monitor2: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "neutron router-list" |
+| | monitor3: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "heat stack-list" |
+| | monitor4: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "cinder list" |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there is one metric: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc050.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the turnoff network interface script with param value |
+| | specified by "interface". |
+| | |
+| | Result: Network interfaces will be turned down. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It turns up the |
+| | network interface of the control node if it is not turned |
+| | up. |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc051.rst b/docs/testing/user/userguide/opnfv_yardstick_tc051.rst
new file mode 100644
index 000000000..3402ccd92
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc051.rst
@@ -0,0 +1,117 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC051
+*************************************
+
++-----------------------------------------------------------------------------+
+|OpenStack Controller Node CPU Overload High Availability |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC051: OpenStack Controller Node CPU |
+| | Overload High Availability |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of control |
+| | node. When the CPU usage of a specified controller node is |
+| | stressed to 100%, which breaks down the Openstack services |
+| | on this node. These Openstack service should able to be |
+| | accessed by other controller nodes, and the services on |
+| | failed controller node should be isolated. |
++--------------+--------------------------------------------------------------+
+|test method | This test case stresses the CPU uasge of a specified control |
+| | node to 100%, then checks whether all services provided by |
+| | the environment are OK with some monitor tools. |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "stress-cpu" is |
+| | needed. This attacker includes two parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "stress-cpu" in |
+| | this test case. |
+| | 2) host: which is the name of a control node being attacked. |
+| | e.g. |
+| | -fault_type: "stress-cpu" |
+| | -host: node1 |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, the monitor named "openstack-cmd" is |
+| | needed. The monitor needs needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request |
+| | |
+| | There are four instance of the "openstack-cmd" monitor: |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "nova image-list" |
+| | monitor2: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "neutron router-list" |
+| | monitor3: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "heat stack-list" |
+| | monitor4: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "cinder list" |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there is one metric: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc051.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the stress cpu script on the host. |
+| | |
+| | Result: The CPU usage of the host will be stressed to 100%. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It kills the |
+| | process that stresses the CPU usage. |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc052.rst b/docs/testing/user/userguide/opnfv_yardstick_tc052.rst
new file mode 100644
index 000000000..9514b6819
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc052.rst
@@ -0,0 +1,141 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC052
+*************************************
+
++-----------------------------------------------------------------------------+
+|OpenStack Controller Node Disk I/O Block High Availability |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC052: OpenStack Controller Node Disk I/O |
+| | Block High Availability |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of control |
+| | node. When the disk I/O of a specified disk is blocked, |
+| | which breaks down the Openstack services on this node. Read |
+| | and write services should still be accessed by other |
+| | controller nodes, and the services on failed controller node |
+| | should be isolated. |
++--------------+--------------------------------------------------------------+
+|test method | This test case blocks the disk I/O of a specified control |
+| | node, then checks whether the services that need to read or |
+| | wirte the disk of the control node are OK with some monitor |
+| | tools. |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "disk-block" is |
+| | needed. This attacker includes two parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "disk-block" in this |
+| | test case. |
+| | 2) host: which is the name of a control node being attacked. |
+| | e.g. |
+| | -fault_type: "disk-block" |
+| | -host: node1 |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scripts. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request. |
+| | |
+| | e.g. |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "nova flavor-list" |
+| | |
+| | 2. the second monitor verifies the read and write function |
+| | by a "operation" and a "result checker". |
+| | the "operation" have two parameters: |
+| | 1) operation_type: which is used for finding the operation |
+| | class and related scripts. |
+| | 2) action_parameter: parameters for the operation. |
+| | the "result checker" have three parameters: |
+| | 1) checker_type: which is used for finding the reuslt |
+| | checker class and realted scripts. |
+| | 2) expectedValue: the expected value for the output of the |
+| | checker script. |
+| | 3) condition: whether the expected value is in the output of |
+| | checker script or is totally same with the output. |
+| | |
+| | In this case, the "operation" adds a flavor and the "result |
+| | checker" checks whether ths flavor is created. Their |
+| | parameters show as follows: |
+| | operation: |
+| | -operation_type: "nova-create-flavor" |
+| | -action_parameter: |
+| | flavorconfig: "test-001 test-001 100 1 1" |
+| | result checker: |
+| | -checker_type: "check-flavor" |
+| | -expectedValue: "test-001" |
+| | -condition: "in" |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there is one metric: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc052.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | do attacker: connect the host through SSH, and then execute |
+| | the block disk I/O script on the host. |
+| | |
+| | Result: The disk I/O of the host will be blocked |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | do operation: add a flavor |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | do result checker: check whether the falvor is created |
+| | |
++--------------+--------------------------------------------------------------+
+|step 5 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 6 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It excutes the |
+| | release disk I/O script to release the blocked I/O. |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails if monnitor SLA is not passed or the result checker is |
+| | not passed, or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc053.rst b/docs/testing/user/userguide/opnfv_yardstick_tc053.rst
new file mode 100644
index 000000000..3c6bbc628
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc053.rst
@@ -0,0 +1,142 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC053
+*************************************
+
++-----------------------------------------------------------------------------+
+|OpenStack Controller Load Balance Service High Availability |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC053: OpenStack Controller Load Balance |
+| | Service High Availability |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability of the |
+| | load balance service(current is HAProxy) that supports |
+| | OpenStack on controller node. When the load balance service |
+| | of a specified controller node is killed, whether other load |
+| | balancers on other controller nodes will work, and whether |
+| | the controller node will restart the load balancer are |
+| | checked. |
++--------------+--------------------------------------------------------------+
+|test method | This test case kills the processes of load balance service |
+| | on a selected control node, then checks whether the request |
+| | of the related Openstack command is OK and the killed |
+| | processes are recovered. |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "kill-process" is |
+| | needed. This attacker includes three parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "kill-process" in this |
+| | test case. |
+| | 2) process_name: which is the process name of the specified |
+| | OpenStack service. If there are multiple processes use the |
+| | same name on the host, all of them are killed by this |
+| | attacker. |
+| | In this case. This parameter should always set to "swift- |
+| | proxy". |
+| | 3) host: which is the name of a control node being attacked. |
+| | |
+| | e.g. |
+| | -fault_type: "kill-process" |
+| | -process_name: "haproxy" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scritps. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request. |
+| | |
+| | 2. the "process" monitor check whether a process is running |
+| | on a specific node, which needs three parameters: |
+| | 1) monitor_type: which used for finding the monitor class |
+| | and related scripts. It should be always set to "process" |
+| | for this monitor. |
+| | 2) process_name: which is the process name for monitor |
+| | 3) host: which is the name of the node runing the process |
+| | In this case, the command_name of monitor1 should be |
+| | services that is supported by load balancer and the process- |
+| | name of monitor2 should be "haproxy", for example: |
+| | |
+| | e.g. |
+| | monitor1: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "nova image-list" |
+| | monitor2: |
+| | -monitor_type: "process" |
+| | -process_name: "haproxy" |
+| | -host: node1 |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there are two metrics: |
+| | 1)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
+| | 2)process_recover_time: which indicates the maximun time |
+| | (seconds) from the process being killed to recovered |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc053.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the kill process script with param value specified by |
+| | "process_name" |
+| | |
+| | Result: Process will be killed. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It will check |
+| | the status of the specified process on the host, and restart |
+| | the process if it is not running for next test cases. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc054.rst b/docs/testing/user/userguide/opnfv_yardstick_tc054.rst
new file mode 100644
index 000000000..7f92be2bc
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc054.rst
@@ -0,0 +1,125 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Yin Kanglin and others.
+.. 14_ykl@tongji.edu.cn
+
+*************************************
+Yardstick Test Case Description TC054
+*************************************
+
++-----------------------------------------------------------------------------+
+|OpenStack Virtual IP High Availability |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC054: OpenStack Virtual IP High |
+| | Availability |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will verify the high availability for virtual |
+| | ip in the environment. When master node of virtual ip is |
+| | abnormally shutdown, connection to virtual ip and |
+| | the services binded to the virtual IP it should be OK. |
++--------------+--------------------------------------------------------------+
+|test method | This test case shutdowns the virtual IP master node with |
+| | some fault injection tools, then checks whether virtual ips |
+| | can be pinged and services binded to virtual ip are OK with |
+| | some monitor tools. |
++--------------+--------------------------------------------------------------+
+|attackers | In this test case, an attacker called "control-shutdown" is |
+| | needed. This attacker includes two parameters: |
+| | 1) fault_type: which is used for finding the attacker's |
+| | scripts. It should be always set to "control-shutdown" in |
+| | this test case. |
+| | 2) host: which is the name of a control node being attacked. |
+| | |
+| | In this case the host should be the virtual ip master node, |
+| | that means the host ip is the virtual ip, for exapmle: |
+| | -fault_type: "control-shutdown" |
+| | -host: node1(the VIP Master node) |
++--------------+--------------------------------------------------------------+
+|monitors | In this test case, two kinds of monitor are needed: |
+| | 1. the "ip_status" monitor that pings a specific ip to check |
+| | the connectivity of this ip, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scripts. It should be always set to "ip_status" |
+| | for this monitor. |
+| | 2) ip_address: The ip to be pinged. In this case, ip_address |
+| | should be the virtual IP. |
+| | |
+| | 2. the "openstack-cmd" monitor constantly request a specific |
+| | Openstack command, which needs two parameters: |
+| | 1) monitor_type: which is used for finding the monitor class |
+| | and related scripts. It should be always set to |
+| | "openstack-cmd" for this monitor. |
+| | 2) command_name: which is the command name used for request. |
+| | |
+| | e.g. |
+| | monitor1: |
+| | -monitor_type: "ip_status" |
+| | -host: 192.168.0.2 |
+| | monitor2: |
+| | -monitor_type: "openstack-cmd" |
+| | -command_name: "nova image-list" |
+| | |
++--------------+--------------------------------------------------------------+
+|metrics | In this test case, there are two metrics: |
+| | 1) ping_outage_time: which-indicates the maximum outage time |
+| | to ping the specified host. |
+| | 2)service_outage_time: which indicates the maximum outage |
+| | time (seconds) of the specified Openstack command request. |
++--------------+--------------------------------------------------------------+
+|test tool | Developed by the project. Please see folder: |
+| | "yardstick/benchmark/scenarios/availability/ha_tools" |
+| | |
++--------------+--------------------------------------------------------------+
+|references | ETSI NFV REL001 |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | This test case needs two configuration files: |
+| | 1) test case file: opnfv_yardstick_tc054.yaml |
+| | -Attackers: see above "attackers" discription |
+| | -waiting_time: which is the time (seconds) from the process |
+| | being killed to stoping monitors the monitors |
+| | -Monitors: see above "monitors" discription |
+| | -SLA: see above "metrics" discription |
+| | |
+| | 2)POD file: pod.yaml |
+| | The POD configuration should record on pod.yaml first. |
+| | the "host" item in this test case will use the node name in |
+| | the pod.yaml. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | start monitors: |
+| | each monitor will run with independently process |
+| | |
+| | Result: The monitor info will be collected. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | do attacker: connect the host through SSH, and then execute |
+| | the shutdown script on the VIP master node. |
+| | |
+| | Result: VIP master node will be shutdown |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | stop monitors after a period of time specified by |
+| | "waiting_time" |
+| | |
+| | Result: The monitor info will be aggregated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 4 | verify the SLA |
+| | |
+| | Result: The test case is passed or not. |
+| | |
++--------------+--------------------------------------------------------------+
+|post-action | It is the action when the test cases exist. It restarts the |
+| | original VIP master node if it is not restarted. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc055.rst b/docs/testing/user/userguide/opnfv_yardstick_tc055.rst
new file mode 100644
index 000000000..c861ca90c
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc055.rst
@@ -0,0 +1,67 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC055
+*************************************
+
+.. _/proc/cpuinfo: http://www.linfo.org/proc_cpuinfo.html
+
++-----------------------------------------------------------------------------+
+|Compute Capacity |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC055_Compute Capacity |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of cpus, number of cores, number of threads, available|
+| | memory size and total cache size. |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS compute capacity with regards to |
+| | hardware specification, including number of cpus, number of |
+| | cores, number of threads, available memory size and total |
+| | cache size. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc055.yaml |
+| | |
+| | There is are no additional configurations to be set for this |
+| | TC. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | /proc/cpuinfo |
+| | |
+| | this TC uses /proc/cpuinfo as source to produce compute |
+| | capacity output. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | /proc/cpuinfo_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | None. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | No POD specific requirements have been identified. |
+|conditions | |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The hosts are installed, TC is invoked and logs are produced |
+| | and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. Hardware specification are fetched and stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc061.rst b/docs/testing/user/userguide/opnfv_yardstick_tc061.rst
new file mode 100644
index 000000000..1d424414e
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc061.rst
@@ -0,0 +1,88 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC061
+*************************************
+
+.. _man-pages: http://linux.die.net/man/1/sar
+
++-----------------------------------------------------------------------------+
+|Network Utilization |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC061_Network Utilization |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Network utilization |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS network capability with regards to |
+| | network utilization, including Total number of packets |
+| | received per second, Total number of packets transmitted per |
+| | second, Total number of kilobytes received per second, Total |
+| | number of kilobytes transmitted per second, Number of |
+| | compressed packets received per second (for cslip etc.), |
+| | Number of compressed packets transmitted per second, Number |
+| | of multicast packets received per second, Utilization |
+| | percentage of the network interface. |
+| | This test case should be run in parallel to other Yardstick |
+| | test cases and not run as a stand-alone test case. |
+| | Measure the network usage statistics from the network devices|
+| | Average, minimum and maximun values are obtained. |
+| | The purpose is also to be able to spot trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | File: netutilization.yaml (in the 'samples' directory) |
+| | |
+| | * interval: 1 - repeat, pausing every 1 seconds in-between. |
+| | * count: 1 - display statistics 1 times, then exit. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | sar |
+| | |
+| | The sar command writes to standard output the contents of |
+| | selected cumulative activity counters in the operating |
+| | system. |
+| | sar is normally part of a Linux distribution, hence it |
+| | doesn't needs to be installed. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | man-pages_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * interval; |
+| | * count; |
+| | * runner Iteration and intervals. |
+| | |
+| | There are default values for each above-mentioned option. |
+| | Run in background with other test cases. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with sar included in the image. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The host is installed as client. The related TC, or TCs, is |
+| | invoked and sar logs are produced and stored. |
+| | |
+| | Result: logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. Network utilization results are fetched and stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc063.rst b/docs/testing/user/userguide/opnfv_yardstick_tc063.rst
new file mode 100644
index 000000000..a77653aa5
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc063.rst
@@ -0,0 +1,81 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC063
+*************************************
+
+.. _iostat: http://linux.die.net/man/1/iostat
+.. _fdisk: http://www.tldp.org/HOWTO/Partition/fdisk_partitioning.html
+
++-----------------------------------------------------------------------------+
+|Storage Capacity |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC063_Storage Capacity |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Storage/disk size, block size |
+| | Disk Utilization |
++--------------+--------------------------------------------------------------+
+|test purpose | This test case will check the parameters which could decide |
+| | several models and each model has its specified task to |
+| | measure. The test purposes are to measure disk size, block |
+| | size and disk utilization. With the test results, we could |
+| | evaluate the storage capacity of the host. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc063.yaml |
+| | |
+| |* test_type: "disk_size" |
+| |* runner: |
+| | type: Iteration |
+| | iterations: 1 - test is run 1 time iteratively. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | fdisk |
+| | A command-line utility that provides disk partitioning |
+| | functions |
+| | |
+| | iostat |
+| | This is a computer system monitor tool used to collect and |
+| | show operating system storage input and output statistics. |
++--------------+--------------------------------------------------------------+
+|references | iostat_ |
+| | fdisk_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * test_type: "disk size", "block size", "disk utilization" |
+| | * interval: 1 - how ofter to stat disk utilization |
+| | type: int |
+| | unit: seconds |
+| | * count: 15 - how many times to stat disk utilization |
+| | type: int |
+| | unit: na |
+| | There are default values for each above-mentioned option. |
+| | Run in background with other test cases. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | Output the specific storage capacity of disk information as |
+| | the sequence into file. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The pod is available and the hosts are installed. Node5 is |
+| | used and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc069.rst b/docs/testing/user/userguide/opnfv_yardstick_tc069.rst
new file mode 100644
index 000000000..af0e64fbf
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc069.rst
@@ -0,0 +1,100 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC069
+*************************************
+
+.. _RAMspeed: http://alasir.com/software/ramspeed/
+
+.. table::
+ :class: longtable
+
++-----------------------------------------------------------------------------+
+|Memory Bandwidth |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC069_Memory Bandwidth |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Megabyte per second (MBps) |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS compute performance with regards to |
+| | memory bandwidth. |
+| | Measure the maximum possible cache and memory performance |
+| | while reading and writing certain blocks of data (starting |
+| | from 1Kb and further in power of 2) continuously through ALU |
+| | and FPU respectively. |
+| | Measure different aspects of memory performance via |
+| | synthetic simulations. Each simulation consists of four |
+| | performances (Copy, Scale, Add, Triad). |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | File: opnfv_yardstick_tc069.yaml |
+| | |
+| | * SLA (optional): 7000 (MBps) min_bandwidth: The minimum |
+| | amount of memory bandwidth that is accepted. |
+| | * type_id: 1 - runs a specified benchmark |
+| | (by an ID number): |
+| | 1 -- INTmark [writing] 4 -- FLOATmark [writing] |
+| | 2 -- INTmark [reading] 5 -- FLOATmark [reading] |
+| | 3 -- INTmem 6 -- FLOATmem |
+| | * block_size: 64 Megabytes - the maximum block |
+| | size per array. |
+| | * load: 32 Gigabytes - the amount of data load per pass. |
+| | * iterations: 5 - test is run 5 times iteratively. |
+| | * interval: 1 - there is 1 second delay between each |
+| | iteration. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | RAMspeed |
+| | |
+| | RAMspeed is a free open source command line utility to |
+| | measure cache and memory performance of computer systems. |
+| | RAMspeed is not always part of a Linux distribution, hence |
+| | it needs to be installed in the test image. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | RAMspeed_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * benchmark operations (such as INTmark [writing], |
+| | INTmark [reading], FLOATmark [writing], |
+| | FLOATmark [reading], INTmem, FLOATmem); |
+| | * block size per array; |
+| | * load per pass; |
+| | * number of batch run iterations; |
+| | * iterations and intervals. |
+| | |
+| | There are default values for each above-mentioned option. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with RAmspeed included in the image. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The host is installed as client. RAMspeed is invoked and |
+| | logs are produced and stored. |
+| | |
+| | Result: logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Test fails if the measured memory bandwidth is below the SLA |
+| | value or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc070.rst b/docs/testing/user/userguide/opnfv_yardstick_tc070.rst
new file mode 100644
index 000000000..64fcc0c91
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc070.rst
@@ -0,0 +1,110 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC070
+*************************************
+
+.. _cirros: https://download.cirros-cloud.net
+.. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+.. _free: http://manpages.ubuntu.com/manpages/trusty/en/man1/free.1.html
+
++-----------------------------------------------------------------------------+
+|Latency, Memory Utilization, Throughput, Packet Loss |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC070_Latency, Memory Utilization, |
+| | Throughput,Packet Loss |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of flows, latency, throughput, Memory Utilization, |
+| | packet loss |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS network performance with regards to |
+| | flows and throughput, such as if and how different amounts |
+| | of flows matter for the throughput between hosts on different|
+| | compute blades. Typically e.g. the performance of a vSwitch |
+| | depends on the number of flows running through it. Also |
+| | performance of other equipment or entities can depend |
+| | on the number of flows or the packet sizes used. |
+| | The purpose is also to be able to spot trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc070.yaml |
+| | |
+| | Packet size: 64 bytes |
+| | Number of ports: 1, 10, 50, 100, 300, 500, 750 and 1000. |
+| | The amount configured ports map from 2 up to 1001000 flows, |
+| | respectively. Each port amount is run two times, for 20 |
+| | seconds each. Then the next port_amount is run, and so on. |
+| | During the test Memory Utilization on both client and server,|
+| | and the network latency between the client and server are |
+| | measured. |
+| | The client and server are distributed on different HW. |
+| | For SLA max_ppm is set to 1000. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | pktgen |
+| | |
+| | Pktgen is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Glance |
+| | image. |
+| | (As an example see the /yardstick/tools/ directory for how |
+| | to generate a Linux image with pktgen included.) |
+| | |
+| | ping |
+| | |
+| | Ping is normally part of any Linux distribution, hence it |
+| | doesn't need to be installed. It is also part of the |
+| | Yardstick Glance image. |
+| | (For example also a cirros_ image can be downloaded, it |
+| | includes ping) |
+| | |
+| | free |
+| | |
+| | free provides information about unused and used memory and |
+| | swap space on any computer running Linux or another Unix-like|
+| | operating system. |
+| | free is normally part of a Linux distribution, hence it |
+| | doesn't needs to be installed. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Ping and free man pages |
+| | |
+| | pktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different packet sizes, amount |
+| | of flows and test duration. Default values exist. |
+| | |
+| | SLA (optional): max_ppm: The number of packets per million |
+| | packets sent that are acceptable to lose, not received. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with pktgen included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The hosts are installed, as server and client. pktgen is |
+| | invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc071.rst b/docs/testing/user/userguide/opnfv_yardstick_tc071.rst
new file mode 100644
index 000000000..673480b55
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc071.rst
@@ -0,0 +1,109 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC071
+*************************************
+
+.. _cirros: https://download.cirros-cloud.net
+.. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+.. _cachestat: https://github.com/brendangregg/perf-tools/tree/master/fs
+
++-----------------------------------------------------------------------------+
+|Latency, Cache Utilization, Throughput, Packet Loss |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC071_Latency, Cache Utilization, |
+| | Throughput,Packet Loss |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of flows, latency, throughput, Cache Utilization, |
+| | packet loss |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS network performance with regards to |
+| | flows and throughput, such as if and how different amounts |
+| | of flows matter for the throughput between hosts on different|
+| | compute blades. Typically e.g. the performance of a vSwitch |
+| | depends on the number of flows running through it. Also |
+| | performance of other equipment or entities can depend |
+| | on the number of flows or the packet sizes used. |
+| | The purpose is also to be able to spot trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc071.yaml |
+| | |
+| | Packet size: 64 bytes |
+| | Number of ports: 1, 10, 50, 100, 300, 500, 750 and 1000. |
+| | The amount configured ports map from 2 up to 1001000 flows, |
+| | respectively. Each port amount is run two times, for 20 |
+| | seconds each. Then the next port_amount is run, and so on. |
+| | During the test Cache Utilization on both client and server, |
+| | and the network latency between the client and server are |
+| | measured. |
+| | The client and server are distributed on different HW. |
+| | For SLA max_ppm is set to 1000. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | pktgen |
+| | |
+| | Pktgen is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Glance |
+| | image. |
+| | (As an example see the /yardstick/tools/ directory for how |
+| | to generate a Linux image with pktgen included.) |
+| | |
+| | ping |
+| | |
+| | Ping is normally part of any Linux distribution, hence it |
+| | doesn't need to be installed. It is also part of the |
+| | Yardstick Glance image. |
+| | (For example also a cirros_ image can be downloaded, it |
+| | includes ping) |
+| | |
+| | cachestat |
+| | |
+| | cachestat is not always part of a Linux distribution, hence |
+| | it needs to be installed. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Ping man pages |
+| | |
+| | pktgen_ |
+| | |
+| | cachestat_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different packet sizes, amount |
+| | of flows and test duration. Default values exist. |
+| | |
+| | SLA (optional): max_ppm: The number of packets per million |
+| | packets sent that are acceptable to lose, not received. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with pktgen included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The hosts are installed, as server and client. pktgen is |
+| | invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc072.rst b/docs/testing/user/userguide/opnfv_yardstick_tc072.rst
new file mode 100644
index 000000000..2e7ee057c
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc072.rst
@@ -0,0 +1,110 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC072
+*************************************
+
+.. _cirros: https://download.cirros-cloud.net
+.. _pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+.. _sar: http://linux.die.net/man/1/sar
+
++-----------------------------------------------------------------------------+
+|Latency, Network Utilization, Throughput, Packet Loss |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC072_Latency, Network Utilization, |
+| | Throughput,Packet Loss |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of flows, latency, throughput, Network Utilization, |
+| | packet loss |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS network performance with regards to |
+| | flows and throughput, such as if and how different amounts |
+| | of flows matter for the throughput between hosts on different|
+| | compute blades. Typically e.g. the performance of a vSwitch |
+| | depends on the number of flows running through it. Also |
+| | performance of other equipment or entities can depend |
+| | on the number of flows or the packet sizes used. |
+| | The purpose is also to be able to spot trends. |
+| | Test results, graphs and similar shall be stored for |
+| | comparison reasons and product evolution understanding |
+| | between different OPNFV versions and/or configurations. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc072.yaml |
+| | |
+| | Packet size: 64 bytes |
+| | Number of ports: 1, 10, 50, 100, 300, 500, 750 and 1000. |
+| | The amount configured ports map from 2 up to 1001000 flows, |
+| | respectively. Each port amount is run two times, for 20 |
+| | seconds each. Then the next port_amount is run, and so on. |
+| | During the test Network Utilization on both client and |
+| | server, and the network latency between the client and server|
+| | are measured. |
+| | The client and server are distributed on different HW. |
+| | For SLA max_ppm is set to 1000. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | pktgen |
+| | |
+| | Pktgen is not always part of a Linux distribution, hence it |
+| | needs to be installed. It is part of the Yardstick Glance |
+| | image. |
+| | (As an example see the /yardstick/tools/ directory for how |
+| | to generate a Linux image with pktgen included.) |
+| | |
+| | ping |
+| | |
+| | Ping is normally part of any Linux distribution, hence it |
+| | doesn't need to be installed. It is also part of the |
+| | Yardstick Glance image. |
+| | (For example also a cirros_ image can be downloaded, it |
+| | includes ping) |
+| | |
+| | sar |
+| | |
+| | The sar command writes to standard output the contents of |
+| | selected cumulative activity counters in the operating |
+| | system. |
+| | sar is normally part of a Linux distribution, hence it |
+| | doesn't needs to be installed. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Ping and sar man pages |
+| | |
+| | pktgen_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different packet sizes, amount |
+| | of flows and test duration. Default values exist. |
+| | |
+| | SLA (optional): max_ppm: The number of packets per million |
+| | packets sent that are acceptable to lose, not received. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The test case image needs to be installed into Glance |
+|conditions | with pktgen included in it. |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The hosts are installed, as server and client. pktgen is |
+| | invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc073.rst b/docs/testing/user/userguide/opnfv_yardstick_tc073.rst
new file mode 100644
index 000000000..ad4526405
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc073.rst
@@ -0,0 +1,81 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC073
+*************************************
+
+.. _netperf: http://www.netperf.org/netperf/training/Netperf.html
+
++-----------------------------------------------------------------------------+
+|Throughput per NFVI node test |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC073_Network latency and throughput between |
+| | nodes |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Network latency and throughput |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the IaaS network performance with regards to |
+| | flows and throughput, such as if and how different amounts |
+| | of packet sizes and flows matter for the throughput between |
+| | nodes in one pod. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc073.yaml |
+| | |
+| | Packet size: default 1024 bytes. |
+| | |
+| | Test length: default 20 seconds. |
+| | |
+| | The client and server are distributed on different nodes. |
+| | |
+| | For SLA max_mean_latency is set to 100. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | netperf_ |
+| | Netperf is a software application that provides network |
+| | bandwidth testing between two hosts on a network. It |
+| | supports Unix domain sockets, TCP, SCTP, DLPI and UDP via |
+| | BSD Sockets. Netperf provides a number of predefined tests |
+| | e.g. to measure bulk (unidirectional) data transfer or |
+| | request response performance. |
+| | (netperf is not always part of a Linux distribution, hence |
+| | it needs to be installed.) |
+| | |
++--------------+--------------------------------------------------------------+
+|references | netperf Man pages |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different packet sizes and |
+| | test duration. Default values exist. |
+| | |
+| | SLA (optional): max_mean_latency |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | The POD can be reached by external ip and logged on via ssh |
+|conditions | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | Install netperf tool on each specified node, one is as the |
+| | server, and the other as the client. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Log on to the client node and use the netperf command to |
+| | execute the network performance test |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | The throughput results stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Fails only if SLA is not passed, or if there is a test case |
+| | execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc074.rst b/docs/testing/user/userguide/opnfv_yardstick_tc074.rst
new file mode 100644
index 000000000..92cd51439
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc074.rst
@@ -0,0 +1,137 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC074
+*************************************
+
+.. _Storperf: https://wiki.opnfv.org/display/storperf/Storperf
+
++-----------------------------------------------------------------------------+
+|Storperf |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC074_Storperf |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Storage performance |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | Storperf integration with yardstick. The purpose of StorPerf |
+| | is to provide a tool to measure block and object storage |
+| | performance in an NFVI. When complemented with a |
+| | characterization of typical VF storage performance |
+| | requirements, it can provide pass/fail thresholds for test, |
+| | staging, and production NFVI environments. |
+| | |
+| | The benchmarks developed for block and object storage will |
+| | be sufficiently varied to provide a good preview of expected |
+| | storage performance behavior for any type of VNF workload. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc074.yaml |
+| | |
+| | * agent_count: 1 - the number of VMs to be created |
+| | * agent_image: "Ubuntu-14.04" - image used for creating VMs |
+| | * public_network: "ext-net" - name of public network |
+| | * volume_size: 2 - cinder volume size |
+| | * block_sizes: "4096" - data block size |
+| | * queue_depths: "4" |
+| | * StorPerf_ip: "192.168.200.2" |
+| | * query_interval: 10 - state query interval |
+| | * timeout: 600 - maximum allowed job time |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | Storperf_ |
+| | |
+| | StorPerf is a tool to measure block and object storage |
+| | performance in an NFVI. |
+| | |
+| | StorPerf is delivered as a Docker container from |
+| | https://hub.docker.com/r/opnfv/storperf/tags/. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Storperf_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * agent_count |
+| | * volume_size |
+| | * block_sizes |
+| | * queue_depths |
+| | * query_interval |
+| | * timeout |
+| | * target=[device or path] |
+| | The path to either an attached storage device |
+| | (/dev/vdb, etc) or a directory path (/opt/storperf) that |
+| | will be used to execute the performance test. In the case |
+| | of a device, the entire device will be used. If not |
+| | specified, the current directory will be used. |
+| | * workload=[workload module] |
+| | If not specified, the default is to run all workloads. The |
+| | workload types are: |
+| | - rs: 100% Read, sequential data |
+| | - ws: 100% Write, sequential data |
+| | - rr: 100% Read, random access |
+| | - wr: 100% Write, random access |
+| | - rw: 70% Read / 30% write, random access |
+| | * nossd: Do not perform SSD style preconditioning. |
+| | * nowarm: Do not perform a warmup prior to |
+| | measurements. |
+| | * report= [job_id] |
+| | Query the status of the supplied job_id and report on |
+| | metrics. If a workload is supplied, will report on only |
+| | that subset. |
+| | |
+| | There are default values for each above-mentioned option. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | If you do not have an Ubuntu 14.04 image in Glance, you will |
+|conditions | need to add one. A key pair for launching agents is also |
+| | required. |
+| | |
+| | Storperf is required to be installed in the environment. |
+| | There are two possible methods for Storperf installation: |
+| | Run container on Jump Host |
+| | Run container in a VM |
+| | |
+| | Running StorPerf on Jump Host |
+| | Requirements: |
+| | - Docker must be installed |
+| | - Jump Host must have access to the OpenStack Controller |
+| | API |
+| | - Jump Host must have internet connectivity for |
+| | downloading docker image |
+| | - Enough floating IPs must be available to match your |
+| | agent count |
+| | |
+| | Running StorPerf in a VM |
+| | Requirements: |
+| | - VM has docker installed |
+| | - VM has OpenStack Controller credentials and can |
+| | communicate with the Controller API |
+| | - VM has internet connectivity for downloading the |
+| | docker image |
+| | - Enough floating IPs must be available to match your |
+| | agent count |
+| | |
+| | No POD specific requirements have been identified. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The Storperf is installed and Ubuntu 14.04 image is stored |
+| | in glance. TC is invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. Storage performance results are fetched and stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc075.rst b/docs/testing/user/userguide/opnfv_yardstick_tc075.rst
new file mode 100644
index 000000000..a6ff34447
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc075.rst
@@ -0,0 +1,60 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC075
+*************************************
+
+
++-----------------------------------------------------------------------------+
+|Network Capacity and Scale Testing |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC075_Network_Capacity_and_Scale_testing |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | Number of connections, Number of frames sent/received |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To evaluate the network capacity and scale with regards to |
+| | connections and frmaes. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc075.yaml |
+| | |
+| | There is no additional configuration to be set for this TC. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | netstar |
+| | |
+| | Netstat is normally part of any Linux distribution, hence it |
+| | doesn't need to be installed. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Netstat man page |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | This test case is mainly for evaluating network performance. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre_test | Each pod node must have netstat included in it. |
+|conditions | |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The pod is available. |
+| | Netstat is invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. Number of connections and frames are fetched and |
+| | stored. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/opnfv_yardstick_tc076.rst b/docs/testing/user/userguide/opnfv_yardstick_tc076.rst
new file mode 100644
index 000000000..ac7bde794
--- /dev/null
+++ b/docs/testing/user/userguide/opnfv_yardstick_tc076.rst
@@ -0,0 +1,61 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC076
+*************************************
+
+
++-----------------------------------------------------------------------------+
+|Monitor Network Metrics |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC076_Monitor_Network_Metrics |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | IP datagram error rate, ICMP message error rate, |
+| | TCP segment error rate and UDP datagram error rate |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | Monitor network metrics provided by the kernel in a host and |
+| | calculate IP datagram error rate, ICMP message error rate, |
+| | TCP segment error rate and UDP datagram error rate. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc076.yaml |
+| | |
+| | There is no additional configuration to be set for this TC. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | nstat |
+| | |
+| | nstat is a simple tool to monitor kernel snmp counters and |
+| | network interface statistics. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | nstat man page |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | This test case is mainly for monitoring network metrics. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre_test | |
+|conditions | |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The pod is available. |
+| | Nstat is invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/docs/testing/user/userguide/references.rst b/docs/testing/user/userguide/references.rst
new file mode 100644
index 000000000..05729ba75
--- /dev/null
+++ b/docs/testing/user/userguide/references.rst
@@ -0,0 +1,60 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+==========
+References
+==========
+
+
+OPNFV
+=====
+
+* Parser wiki: https://wiki.opnfv.org/parser
+* Pharos wiki: https://wiki.opnfv.org/pharos
+* VTC: https://wiki.opnfv.org/vtc
+* Yardstick CI: https://build.opnfv.org/ci/view/yardstick/
+* Yardstick and ETSI TST001 presentation: https://wiki.opnfv.org/display/yardstick/Yardstick?preview=%2F2925202%2F2925205%2Fopnfv_summit_-_bridging_opnfv_and_etsi.pdf
+* Yardstick Project presentation: https://wiki.opnfv.org/display/yardstick/Yardstick?preview=%2F2925202%2F2925208%2Fopnfv_summit_-_yardstick_project.pdf
+* Yardstick wiki: https://wiki.opnfv.org/yardstick
+
+References used in Test Cases
+=============================
+
+* cachestat: https://github.com/brendangregg/perf-tools/tree/master/fs
+* cirros-image: https://download.cirros-cloud.net
+* cyclictest: https://rt.wiki.kernel.org/index.php/Cyclictest
+* DPDKpktgen: https://github.com/Pktgen/Pktgen-DPDK/
+* DPDK supported NICs: http://dpdk.org/doc/nics
+* fdisk: http://www.tldp.org/HOWTO/Partition/fdisk_partitioning.html
+* fio: http://www.bluestop.org/fio/HOWTO.txt
+* free: http://manpages.ubuntu.com/manpages/trusty/en/man1/free.1.html
+* iperf3: https://iperf.fr/
+* iostat: http://linux.die.net/man/1/iostat
+* Lmbench man-pages: http://manpages.ubuntu.com/manpages/trusty/lat_mem_rd.8.html
+* Memory bandwidth man-pages: http://manpages.ubuntu.com/manpages/trusty/bw_mem.8.html
+* mpstat man-pages: http://manpages.ubuntu.com/manpages/trusty/man1/mpstat.1.html
+* netperf: http://www.netperf.org/netperf/training/Netperf.html
+* pktgen: https://www.kernel.org/doc/Documentation/networking/pktgen.txt
+* RAMspeed: http://alasir.com/software/ramspeed/
+* sar: http://linux.die.net/man/1/sar
+* SR-IOV: https://wiki.openstack.org/wiki/SR-IOV-Passthrough-For-Networking
+* Storperf: https://wiki.opnfv.org/display/storperf/Storperf
+* unixbench: https://github.com/kdlucas/byte-unixbench/blob/master/UnixBench
+
+
+Research
+========
+
+* NCSRD: http://www.demokritos.gr/?lang=en
+* T-NOVA: http://www.t-nova.eu/
+* T-NOVA Results: http://www.t-nova.eu/results/
+
+Standards
+=========
+
+* ETSI NFV: http://www.etsi.org/technologies-clusters/technologies/nfv
+* ETSI GS-NFV TST 001: http://www.etsi.org/deliver/etsi_gs/NFV-TST/001_099/001/01.01.01_60/gs_NFV-TST001v010101p.pdf
+* RFC2544: https://www.ietf.org/rfc/rfc2544.txt
+
diff --git a/docs/testing/user/userguide/testcase_description_v2_template.rst b/docs/testing/user/userguide/testcase_description_v2_template.rst
new file mode 100644
index 000000000..91c2a7e33
--- /dev/null
+++ b/docs/testing/user/userguide/testcase_description_v2_template.rst
@@ -0,0 +1,64 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Ericsson AB and others.
+
+*************************************
+Yardstick Test Case Description TCXXX
+*************************************
+
++-----------------------------------------------------------------------------+
+|test case slogan e.g. Network Latency |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | e.g. OPNFV_YARDSTICK_TC001_NW Latency |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | what will be measured, e.g. latency |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | describe what is the purpose of the test case |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | what .yaml file to use, state SLA if applicable, state |
+| | test duration, list and describe the scenario options used in|
+| | this TC and also list the options using default values. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | e.g. ping |
+| | |
++--------------+--------------------------------------------------------------+
+|references | e.g. RFCxxx, ETSI-NFVyyy |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | describe variations of the test case which can be |
+| | performend, e.g. run the test for different packet sizes |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | describe configuration in the tool(s) used to perform |
+|conditions | the measurements (e.g. fio, pktgen), POD-specific |
+| | configuration required to enable running the test |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | use this to describe tests that require sveveral steps e.g |
+| | collect logs. |
+| | |
+| | Result: what happens in this step e.g. logs collected |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | remove interface |
+| | |
+| | Result: interface down. |
+| | |
++--------------+--------------------------------------------------------------+
+|step N | what is done in step N |
+| | |
+| | Result: what happens |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | expected behavior, or SLA, pass/fail criteria |
+| | |
++--------------+--------------------------------------------------------------+