aboutsummaryrefslogtreecommitdiffstats
path: root/docs/testing/user/userguide/opnfv_yardstick_tc084.rst
blob: 2e7b28e251ccbea6d61ffc8e34f1bce6168b494c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
.. This work is licensed under a Creative Commons Attribution 4.0 International
.. License.
.. http://creativecommons.org/licenses/by/4.0
.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.

*************************************
Yardstick Test Case Description TC084
*************************************

.. _spec_cpu_2006: https://www.spec.org/cpu2006/

+-----------------------------------------------------------------------------+
|Compute Performance                                                          |
|                                                                             |
+--------------+--------------------------------------------------------------+
|test case id  | OPNFV_YARDSTICK_TC084_SPEC CPU 2006 FOR VM                   |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|metric        | compute-intensive performance                                |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test purpose  | The purpose of TC084 is to evaluate the IaaS compute         |
|              | performance by using SPEC CPU 2006 benchmark. The SPEC CPU   |
|              | 2006 benchmark has several different ways to measure         |
|              | computer performance. One way is to measure how fast the     |
|              | computer completes a single task; this is called a speed     |
|              | measurement. Another way is to measure how many tasks        |
|              | computer can accomplish in a certain amount of time; this is |
|              | called a throughput, capacity or rate measurement.           |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test tool     | SPEC CPU 2006                                                |
|              |                                                              |
|              | The SPEC CPU 2006 benchmark is SPEC's industry-standardized, |
|              | CPU-intensive benchmark suite, stressing a system's          |
|              | processor, memory subsystem and compiler. This benchmark     |
|              | suite includes the SPECint benchmarks and the SPECfp         |
|              | benchmarks. The SPECint 2006 benchmark contains 12 different |
|              | benchmark tests and the SPECfp 2006 benchmark contains 19    |
|              | different benchmark tests.                                   |
|              |                                                              |
|              | SPEC CPU 2006 is not always part of a Linux distribution.    |
|              | SPEC requires that users purchase a license and agree with   |
|              | their terms and conditions. For this test case, users must   |
|              | manually download cpu2006-1.2.iso from the SPEC website and  |
|              | save it under the yardstick/resources folder (e.g. /home/    |
|              | opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso)   |
|              | SPEC CPU® 2006 benchmark is available for purchase via the   |
|              | SPEC order form (https://www.spec.org/order.html).           |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test          | This test case uses SPEC CPU 2006 benchmark to measure       |
|description   | compute-intensive performance of VMs.                        |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|configuration | file: opnfv_yardstick_tc084.yaml                             |
|              |                                                              |
|              | benchmark_subset is set to int.                              |
|              |                                                              |
|              | SLA is not available in this test case.                      |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|applicability | Test can be configured with different:                       |
|              |                                                              |
|              |  * benchmark_subset - a subset of SPEC CPU 2006 benchmarks   |
|              |    to run;                                                   |
|              |  * SPECint_benchmark - a SPECint benchmark to run;           |
|              |  * SPECint_benchmark - a SPECfp benchmark to run;            |
|              |  * output_format - desired report format;                    |
|              |  * runspec_config - SPEC CPU 2006 config file provided to    |
|              |    the runspec binary;                                       |
|              |  * runspec_iterations - the number of benchmark iterations   |
|              |    to execute. For a reportable run, must be 3;              |
|              |  * runspec_tune - tuning to use (base, peak, or all). For a  |
|              |    reportable run, must be either base or all. Reportable    |
|              |    runs do base first, then (optionally) peak;               |
|              |  * runspec_size - size of input data to run (test, train, or |
|              |    ref). Reportable runs ensure that your binaries can       |
|              |    produce correct results with the test and train workloads |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|usability     | This test case is used for executing SPEC CPU 2006 benchmark |
|              | on virtual machines. The SPECint 2006 benchmark takes        |
|              | approximately 5 hours. (The time may vary due to different   |
|              | VM cpu configurations)                                       |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|references    | spec_cpu_2006_                                               |
|              |                                                              |
|              | ETSI-NFV-TST001                                              |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|pre-test      | To run and install SPEC CPU 2006, the following are          |
|conditions    | required:                                                    |
|              |  * For SPECint 2006: Both C99 and C++98 compilers are        |
|              |    installed in VM images;                                   |
|              |  * For SPECfp 2006: All three of C99, C++98 and Fortran-95   |
|              |    compilers installed in VM images;                         |
|              |  * At least 4GB of disk space availabile on VM.              |
|              |                                                              |
|              |  gcc 4.8.* and g++ 4.8.* version have been tested in Ubuntu  |
|              |  14.04, Ubuntu 16.04 and Redhat Enterprise Linux 7.4 image.  |
|              |  Higher gcc and g++ version may cause compiling error.       |
|              |                                                              |
|              |  For more SPEC CPU 2006 dependencies please visit            |
|              |  (https://www.spec.org/cpu2006/Docs/techsupport.html)        |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test sequence | description and expected result                              |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 1        | cpu2006-1.2.iso has been saved under the yardstick/resources |
|              | folder (e.g. /home/opnfv/repos/yardstick/yardstick/resources |
|              | /cpu2006-1.2.iso). Additionally, to use your custom runspec  |
|              | config file you can save it under the yardstick/resources/   |
|              | files folder and specify the config file name in the         |
|              | runspec_config parameter.                                    |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 2        | Upload SPEC CPU 2006 ISO to the target VM using scp and      |
|              | install SPEC CPU 2006.                                       |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 3        | Connect to the target server using SSH.                      |
|              | If custom runspec config file is used, copy this file from   |
|              | yardstick to the target VM via the SSH tunnel.               |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 4        | SPEC CPU 2006 benchmark is invoked and SPEC CPU 2006 metrics |
|              | are generated.                                               |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|step 5        | Text, HTML, CSV, PDF, and Configuration file outputs for the |
|              | SPEC CPU 2006 metrics are fetched from the VM and stored     |
|              | under /tmp/result folder.                                    |
|              |                                                              |
+--------------+--------------------------------------------------------------+
|test verdict  | None. SPEC CPU 2006 results are collected and stored.        |
|              |                                                              |
+--------------+--------------------------------------------------------------+
LLECT=y/g' .config make echo "Load DPDK modules and setup hugepages" modprobe uio mkdir -p "/lib/modules/$(uname -r)/extra" cp -r "kmod/igb_uio.ko" "/lib/modules/$(uname -r)/extra" depmod -a modprobe igb_uio sh -c "echo 'uio\nigb_uio\n' > /etc/modules-load.d/nsb.conf" HUGEPGSZ=$(cat < /proc/meminfo | grep Hugepagesize | cut -d : -f 2 | tr -d ' ') Pages=16 if [[ "$HUGEPGSZ" = "2048kB" ]] ; then Pages=16384 fi grep nr_hugepages /etc/sysctl.conf if [[ "$?" -eq '1' ]] ; then sh -c "echo 'vm.nr_hugepages=$Pages' >> /etc/sysctl.conf" fi echo "echo $Pages > /sys/kernel/mm/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" > .echo_tmp echo "Reserving hugepages" sudo sh .echo_tmp rm -f .echo_tmp service procps start echo "Creating /mnt/huge and mounting as hugetlbfs" sudo mkdir -p /mnt/huge grep -s '/mnt/huge' /proc/mounts > /dev/null if [ $? -ne 0 ] ; then sudo mount -t hugetlbfs nodev /mnt/huge fi popd mv "$REPO_DIR/dpdk-16.07" "$INSTALL_BIN_PATH" rm dpdk-16.07.zip fi export RTE_SDK="$INSTALL_BIN_PATH/dpdk-16.07" export RTE_TARGET=x86_64-native-linuxapp-gcc if [ ! -f "$INSTALL_BIN_PATH/vPE_vnf" ]; then pushd . echo "Building vPE VNF..." cd $INSTALL_BIN_PATH/dpdk-16.07/examples/ip_pipeline/ make clean make cp build/ip_pipeline $INSTALL_BIN_PATH/vPE_vnf popd fi echo "Done" } push_nsb_binary() { if [ ! -d "$INSTALL_BIN_PATH/trex/scripts" ]; then cp -fr "$REPO_DIR/trex" "$INSTALL_BIN_PATH" fi rm -rf "$REPO_DIR/trex" if [ -d "$INSTALL_BIN_PATH/trex" ]; then echo "Setup Environment variables for Test Harness...." PYTHONPATH="$INSTALL_BIN_PATH/trex/scripts/automation/trex_control_plane:$INSTALL_BIN_PATH/trex/scripts/automation/trex_control_plane/stl" PY_PATH=$(grep PYTHONPATH ~/.bash_profile) if [ "$PY_PATH" = "" ] ; then sh -c "echo export PYTHONPATH=$PYTHONPATH >> ~/.bash_profile" > /dev/null else echo "Your ~/.bash_profile already contains a PYTHONPATH definition." echo "Make sure it contains $PYTHONPATH which is required to run TRex" fi fi cp "$REPO_DIR/yardstick/network_services/nfvi/collectd.sh" "$INSTALL_BIN_PATH" cp "$REPO_DIR/yardstick/network_services/nfvi/collectd.conf" "$INSTALL_BIN_PATH" cp "$REPO_DIR/nsb_setup.sh" "$INSTALL_BIN_PATH" echo "Done" } check_installed_files() { if [ ! -f "$INSTALL_BIN_PATH/yardstick_venv/bin/activate" ]; then echo "Installation Error. Failed to create yardstick virtual env..." exit 1 fi if [ ! -d "$INSTALL_BIN_PATH/dpdk-16.07" ]; then echo "Installation Error. Failed to download and install dpdk-16.07..." exit 1 fi if [ ! -d "$INSTALL_BIN_PATH/trex" ]; then echo "Installation Error. Failed to download and configure Trex" exit 1 fi if [ ! -f "$INSTALL_BIN_PATH/vPE_vnf" ]; then echo "Installation Error. vPE VNF not present in install dir $INSTALL_BIN_PATH" exit 1 fi } if [ "$1" == "dpdk" ]; then install_libs install_dpdk else install_libs install_yardstick install_dpdk install_trex push_nsb_binary check_installed_files clear echo "Installation completed..." echo "Virtual Environment : $INSTALL_BIN_PATH/yardstick_venv" echo "Please Refer README.NSB.rst document on how to get started on VNF testing." fi