aboutsummaryrefslogtreecommitdiffstats
path: root/test-requirements.txt
blob: bedd66690d84abfed019766fdd83ac074595b622 (plain) 
1
2
3
4

# this is required for the docs build jobs
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
oslosphinx>=2.5.0 # Apache-2.0
reno>=0.1.1 # Apache-2.0
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ } 
.. This work is licensed under a Creative Commons Attribution 4.0 International
.. License.
.. http://creativecommons.org/licenses/by/4.0
.. (c) OPNFV, Ericsson AB and others.

*************************************
Yardstick Test Case Description TC010
*************************************

.. _lat_mem_rd: http://manpages.ubuntu.com/manpages/trusty/lat_mem_rd.8.html

+-----------------------------------------------------------------------------+
|Memory Latency                                                               |
|                                                                             |
+--------------+--------------------------------------------------------------+
|test case id  | OPNFV_YARDSTICK_TC010_MEMORY LATENCY                         |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|metric        | Memory read latency (nanoseconds)                            |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test purpose  | The purpose of TC010 is to evaluate the IaaS compute         |
|              | performance with regards to memory read latency.             |
|              | It measures the memory read latency for varying memory sizes |
|              | and strides. Whole memory hierarchy is measured.             |
|              |                                                              |
|              | The purpose is also to be able to spot the trends.           |
|              | Test results, graphs and similar shall be stored for         |
|              | comparison reasons and product evolution understanding       |
|              | between different OPNFV versions and/or configurations.      |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test tool     | Lmbench                                                      |
|              |                                                              |
|              | Lmbench is a suite of operating system microbenchmarks. This |
|              | test uses lat_mem_rd tool from that suite including:         |
|              |  * Context switching                                         |
|              |  * Networking: connection establishment, pipe, TCP, UDP, and |
|              |    RPC hot potato                                            |
|              |  * File system creates and deletes                           |
|              |  * Process creation                                          |
|              |  * Signal handling                                           |
|              |  * System call overhead                                      |
|              |  * Memory read latency                                       |
|              |                                                              |
|              | (LMbench is not always part of a Linux distribution, hence   |
|              | it needs to be installed. As an example see the              |
|              | /yardstick/tools/ directory for how to generate a Linux      |
|              | image with LMbench included.)                                |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test          | LMbench lat_mem_rd benchmark measures memory read latency    |
|description   | for varying memory sizes and strides.                        |
|              |                                                              |
|              | The benchmark runs as two nested loops. The outer loop is    |
|              | the stride size. The inner loop is the array size. For each  |
|              | array size, the benchmark creates a ring of pointers that    |
|              | point backward one stride.Traversing the array is done by:   |
|              |                                                              |
|              |         p = (char **)*p;                                     |
|              |                                                              |
|              | in a for loop (the over head of the for loop is not          |
|              | significant; the loop is an unrolled loop 100 loads long).   |
|              | The size of the array varies from 512 bytes to (typically)   |
|              | eight megabytes. For the small sizes, the cache will have an |
|              | effect, and the loads will be much faster. This becomes much |
|              | more apparent when the data is plotted.                      |
|              |                                                              |
|              | Only data accesses are measured; the instruction cache is    |
|              | not measured.                                                |
|              |                                                              |
|              | The results are reported in nanoseconds per load and have    |
|              | been verified accurate to within a few nanoseconds on an SGI |
|              | Indy.                                                        |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|configuration | File: opnfv_yardstick_tc010.yaml                             |
|              |                                                              |
|              | * SLA (max_latency): 30 nanoseconds                          |
|              | * Stride - 128 bytes                                         |
|              | * Stop size - 64 megabytes                                   |
|              | * Iterations: 10 - test is run 10 times iteratively.         |
|              | * Interval: 1 - there is 1 second delay between each         |
|              |   iteration.                                                 |
|              |                                                              |
|              | SLA is optional. The SLA in this test case serves as an      |
|              | example. Considerably lower read latency is expected.        |
|              | However, to cover most configurations, both baremetal and    |
|              | fully virtualized  ones, this value should be possible to    |
|              | achieve and acceptable for black box testing.                |
|              | Many heavy IO applications start to suffer badly if the      |
|              | read latency is higher than this.                            |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|applicability | Test can be configured with different:                       |
|              |                                                              |
|              | * strides;                                                   |
|              | * stop_size;                                                 |
|              | * iterations and intervals.                                  |
|              |                                                              |
|              | Default values exist.                                        |
|              |                                                              |
|              | SLA (optional) : max_latency: The maximum memory latency     |
|              | that is accepted.                                            |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|usability     | This test case is one of Yardstick's generic test. Thus it   |
|              | is runnable on most of the scenarios.                        |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|references    | LMbench lat_mem_rd_                                          |
|              |                                                              |
|              | ETSI-NFV-TST001                                              |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|pre-test      | The test case image needs to be installed into Glance        |
|conditions    | with Lmbench included in the image.                          |
|              |                                                              |
|              | No POD specific requirements have been identified.           |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test sequence | description and expected result                              |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 1        | The host is installed as client. LMbench's lat_mem_rd tool   |
|              | is invoked and logs are produced and stored.                 |
|              |                                                              |
|              | Result: logs are stored.                                     |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 1        | A host VM with LMbench installed is booted.                  |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 2        | Yardstick is connected with the host VM by using ssh.        |
|              | 'lmbench_latency_benchmark' bash script is copyied from Jump |
|              | Host to the host VM via the ssh tunnel.                      |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 3        | 'lmbench_latency_benchmark' script is invoked. LMbench's     |
|              | lat_mem_rd benchmark starts to measures memory read latency  |
|              | for varying memory sizes and strides. Memory read latency    |
|              | are recorded and checked against the SLA. Logs are produced  |
|              | and stored.                                                  |
|              |                                                              |
|              | Result: Logs are stored.                                     |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 4        | The host VM is deleted.                                      |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test verdict  | Test fails if the measured memory latency is above the SLA   |
|              | value or if there is a test case execution problem.          |
|              |                                                              |
+--------------+--------------------------------------------------------------+