trex: Tests with T-Rex in VM

Patch introduces a set of testcases with T-Rex running inside VM.

JIRA: VSPERF-560

Change-Id: I5b0a0220a2d72428ae1210ee9590d39abdfb9c1d
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Signed-off-by: RadoslawX Glombiowski <radoslawx.glombiowski@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com>

12 files changed, 230 insertions, 4 deletions

diff --git a/conf/04_vnf.conf b/conf/04_vnf.conf
index 37fbe2b1..234f11b6 100644
--- a/conf/04_vnf.conf
+++ b/conf/04_vnf.conf
@@ -142,6 +142,8 @@ GUEST_MEMORY = ['2048']
 GUEST_HUGEPAGES_NR = ['1024']
 
 # test-pmd requires 2 VM cores
+# It is also possible to configure GUEST's CPU topology,
+# e.g. GUEST_SMP = ["sockets=1,cores=2"]
 GUEST_SMP = ['2']
 
 # cpu features to the guest, default options provided to pass all available
diff --git a/conf/integration/01c_trex_vm_tests.conf b/conf/integration/01c_trex_vm_tests.conf
new file mode 100644
index 00000000..50982bea
--- /dev/null
+++ b/conf/integration/01c_trex_vm_tests.conf
@@ -0,0 +1,181 @@
+# Copyright 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Testcases in this files use T-Rex running in VM as a traffic generator.
+#
+
+# A set of options passed to the T-Rex stateless server
+_TREX_OPTIONS="--no-scapy-server --no-watchdog --nc"
+_TREX_SERVER_IP="192.168.35.2"
+_TREX_BRIDGE_IP="192.168.35.1"
+_TREX_IP_MASK="24"
+
+# Macro for initialization of T-Rex VM and execution of T-Rex server
+# NOTE: It is expected that T-Rex will run in the LAST VM!
+TREX_VM_INIT = [
+    # configure T-Rex ports, which will be used for traffic generation
+    ['#trex_p1', 'vswitch', 'add_vport', 'int_br0'],
+    ['#trex_p2', 'vswitch', 'add_vport', 'int_br0'],
+    # configure IP access to T-Rex VM
+    ['vswitch', 'add_switch', 'trex_br'],
+    ['#trex_admin', 'vswitch', 'add_vport', 'trex_br'],
+    ['#trex_spare', 'vswitch', 'add_vport', 'trex_br'],    # spare to have even number of NICs
+    ['tools', 'exec_shell', 'sudo ip addr flush dev trex_br'],
+    ['tools', 'exec_shell', 'sudo ip addr add $_TREX_BRIDGE_IP/$_TREX_IP_MASK dev trex_br'],
+    ['tools', 'exec_shell', 'sudo ip link set dev trex_br up'],
+    ['vnf_trex', 'start'],
+    ['vnf_trex', 'execute_and_wait', 'sudo ip addr flush dev eth2'],
+    ['vnf_trex', 'execute_and_wait', 'sudo ip addr add $_TREX_SERVER_IP/$_TREX_IP_MASK dev eth2'],
+    ['vnf_trex', 'execute_and_wait', 'sudo ip link set dev eth2 up'],
+    # preapre system for T-Rex execution
+    ['vnf_trex', 'execute_and_wait', 'sysctl vm.nr_hugepages=$GUEST_HUGEPAGES_NR[-1]'],
+    ['vnf_trex', 'execute_and_wait', 'mkdir -p /dev/hugepages'],
+    ['vnf_trex', 'execute_and_wait', 'mount -t hugetlbfs hugetlbfs /dev/hugepages'],
+    ['vnf_trex', 'execute_and_wait', 'grep -i huge /proc/meminfo'],
+    ['vnf_trex', 'execute_and_wait', 'iptables -F'],
+    # configure passwordless ssh access to VM with T-Rex server
+    ['tools', 'exec_shell', 'sshpass -p $GUEST_PASSWORD[-1] ssh-copy-id -o StrictHostKeyChecking=no $GUEST_USERNAME[-1]@$_TREX_SERVER_IP'],
+    # prepare T-Rex configuration
+    ['vnf_trex', 'execute_and_wait', 'echo \'- port_limit: 2\' > /etc/trex_cfg.yaml'],
+    ['vnf_trex', 'execute_and_wait', 'echo \'  version: 2\' >> /etc/trex_cfg.yaml'],
+    ['vnf_trex', 'execute_and_wait', "echo \"  interfaces: [ '$GUEST_NICS[-1][0]['pci']', '$GUEST_NICS[-1][1]['pci']' ]\" >> /etc/trex_cfg.yaml"],
+    # execute T-Rex stateless server and wait until it is up and running
+    ['vnf_trex', 'execute_and_wait', 'cd $TRAFFICGEN_TREX_BASE_DIR'],
+    ['#trex_pid', 'vnf_trex', 'execute_and_wait', 'nohup sudo bash -c "./t-rex-64 -i $_TREX_OPTIONS" &', '|\[1\] (\d+)$'],
+    ['vnf_trex', 'execute_and_wait', 'echo -ne "Starting T-Rex " ; while ! netstat -nl | grep 4501 &> /dev/null ; do echo -ne "."; sleep 1 ; done; echo', 300],
+]
+
+# T-Rex VM teardown macro
+TREX_VM_FINIT = [
+    ['vnf_trex', 'execute_and_wait', 'sudo kill #STEP[trex_pid][0]'],
+    ['vnf_trex', 'execute_and_wait', 'sudo pkill t-rex-64'],
+    ['vnf_trex', 'stop'],
+    ['vswitch', 'del_port', 'trex_br', '#STEP[trex_admin][0]'],
+    ['vswitch', 'del_port', 'trex_br', '#STEP[trex_spare][0]'],
+    ['tools', 'exec_shell', 'sudo ip link set dev trex_br down'],
+    ['tools', 'exec_shell', 'sudo ip addr flush dev trex_br'],
+    ['vswitch', 'del_switch', 'trex_br'],
+]
+
+# Configure T-Rex loopback test, where traffic from T-Rex is forwarded back via OVS flows
+TREX_VM_TEST = [
+       ['vswitch', 'add_switch', 'int_br0'],
+    ] + TREX_VM_INIT + [
+       ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[trex_p1][1]', 'actions': ['output:#STEP[trex_p2][1]'], 'idle_timeout': '0'}],
+       ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[trex_p2][1]', 'actions': ['output:#STEP[trex_p1][1]'], 'idle_timeout': '0'}],
+       ['trafficgen', 'send_traffic', {}],
+       ['trafficgen', 'get_results'],
+       # cleanup
+    ] + TREX_VM_FINIT
+
+# Configure VM2VM test, where traffic from T-Rex VM is forwarded to 2nd VM with loopback app.
+TREX_VM2VM_TEST = [
+       ['vswitch', 'add_switch', 'int_br0'],
+       ['#vm_p1', 'vswitch', 'add_vport', 'int_br0'],
+       ['#vm_p2', 'vswitch', 'add_vport', 'int_br0'],
+       ['vnf', 'start'],
+    ] + TREX_VM_INIT + [
+       ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[trex_p1][1]', 'actions': ['output:#STEP[vm_p1][1]'], 'idle_timeout': '0'}],
+       ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[vm_p1][1]', 'actions': ['output:#STEP[trex_p1][1]'], 'idle_timeout': '0'}],
+       ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[trex_p2][1]', 'actions': ['output:#STEP[vm_p2][1]'], 'idle_timeout': '0'}],
+       ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[vm_p2][1]', 'actions': ['output:#STEP[trex_p2][1]'], 'idle_timeout': '0'}],
+       ['trafficgen', 'send_traffic', {}],
+       ['trafficgen', 'get_results'],
+       ['vnf', 'stop'],
+       ['vswitch', 'dump_flows', 'int_br0'],
+       # cleanup
+    ] + TREX_VM_FINIT
+
+#
+# A set of tests with T-Rex VM as a traffic generator.
+#
+INTEGRATION_TESTS = INTEGRATION_TESTS + [
+    {
+        "Name": "trex_vm_cont",
+        "Deployment": "clean",
+        "Description": "T-Rex VM - execute RFC2544 Continuous Stream from T-Rex VM and loop it back through Open vSwitch.",
+        "Parameters" : {
+            "TRAFFIC" : {
+                "traffic_type" : "rfc2544_continuous",
+            },
+            'GUEST_LOOPBACK' : ['clean',],
+            'GUEST_NICS_NR' : [4],
+            'GUEST_SMP' : ['sockets=1,cores=3'],
+            'GUEST_CORE_BINDING' : [['6', '7', '8'],],
+            'TRAFFICGEN' : 'Trex',
+            'TRAFFICGEN_TREX_HOST_IP_ADDR' : _TREX_SERVER_IP,
+            'TRAFFICGEN_TREX_BASE_DIR' : '/root/trex/scripts/',
+            'TRAFFICGEN_TREX_USER' : 'root',
+        },
+        "TestSteps": TREX_VM_TEST
+    },
+    {
+        "Name": "trex_vm_tput",
+        "Deployment": "clean",
+        "Description": "T-Rex VM - execute RFC2544 Throughput from T-Rex VM and loop it back through Open vSwitch.",
+        "Parameters" : {
+            "TRAFFIC" : {
+                "traffic_type" : "rfc2544_throughput",
+            },
+            'GUEST_LOOPBACK' : ['clean',],
+            'GUEST_NICS_NR' : [4],
+            'GUEST_SMP' : ['sockets=1,cores=3'],
+            'GUEST_CORE_BINDING' : [['6', '7', '8'],],
+            'TRAFFICGEN' : 'Trex',
+            'TRAFFICGEN_TREX_HOST_IP_ADDR' : _TREX_SERVER_IP,
+            'TRAFFICGEN_TREX_BASE_DIR' : '/root/trex/scripts/',
+            'TRAFFICGEN_TREX_USER' : 'root',
+        },
+        "TestSteps": TREX_VM_TEST
+    },
+    {
+        "Name": "trex_vm2vm_cont",
+        "Deployment": "clean",
+        "Description": "T-Rex VM2VM - execute RFC2544 Continuous Stream from T-Rex VM and loop it back through 2nd VM.",
+        "Parameters" : {
+            'GUEST_LOOPBACK' : ['testpmd', 'clean'],
+            'GUEST_NICS_NR' : [2, 4],
+            'GUEST_SMP' : ['sockets=1,cores=2', 'sockets=1,cores=3'],
+            'GUEST_CORE_BINDING' : [['9', '10'],['6', '7', '8'],],
+            'TRAFFICGEN' : 'Trex',
+            'TRAFFICGEN_TREX_HOST_IP_ADDR' : _TREX_SERVER_IP,
+            'TRAFFICGEN_TREX_BASE_DIR' : '/root/trex/scripts/',
+            'TRAFFICGEN_TREX_USER' : 'root',
+            "TRAFFIC" : {
+                "traffic_type" : "rfc2544_continuous",
+            },
+        },
+        "TestSteps": TREX_VM2VM_TEST,
+    },
+    {
+        "Name": "trex_vm2vm_tput",
+        "Deployment": "clean",
+        "Description": "T-Rex VM2VM - execute RFC2544 Throughput from T-Rex VM and loop it back through 2nd VM.",
+        "Parameters" : {
+            'GUEST_LOOPBACK' : ['testpmd', 'clean'],
+            'GUEST_NICS_NR' : [2, 4],
+            'GUEST_SMP' : ['sockets=1,cores=2', 'sockets=1,cores=3'],
+            'GUEST_CORE_BINDING' : [['9', '10'],['6', '7', '8'],],
+            'TRAFFICGEN' : 'Trex',
+            'TRAFFICGEN_TREX_HOST_IP_ADDR' : _TREX_SERVER_IP,
+            'TRAFFICGEN_TREX_BASE_DIR' : '/root/trex/scripts/',
+            'TRAFFICGEN_TREX_USER' : 'root',
+            "TRAFFIC" : {
+                "traffic_type" : "rfc2544_throughput",
+            },
+        },
+        "TestSteps": TREX_VM2VM_TEST,
+    },
+]
diff --git a/docs/testing/user/userguide/testlist.rst b/docs/testing/user/userguide/testlist.rst
index 2b0e9d7f..21c4b736 100644
--- a/docs/testing/user/userguide/testlist.rst
+++ b/docs/testing/user/userguide/testlist.rst
@@ -388,3 +388,30 @@ ovsdpdk_qos_p2p                          In a p2p setup, ensure when a QoS egres
 ovsdpdk_qos_pvp                          In a pvp setup, ensure when a QoS egress policer is created that the
                                          traffic is limited to the specified rate.
 ======================================== ======================================================================================
+
+T-Rex in VM TestCases
+^^^^^^^^^^^^^^^^^^^^^
+
+A set of functional testcases, which use T-Rex running in VM as a traffic generator.
+These testcases require a VM image with T-Rex server installed. An example of such
+image is a vloop-vnf image with T-Rex available for download at:
+
+http://artifacts.opnfv.org/vswitchperf/vnf/vloop-vnf-ubuntu-16.04_trex_20180209.qcow2
+
+This image can be used for both T-Rex VM and loopback VM in ``vm2vm`` testcases.
+
+**NOTE:** The performance of T-Rex running inside the VM is lower if compared to T-Rex
+execution on bare-metal. The user should perform a calibration of the VM maximum FPS
+capability, to ensure this limitation is understood.
+
+======================================== ======================================================================================
+trex_vm_cont                             T-Rex VM - execute RFC2544 Continuous Stream from T-Rex VM and loop
+                                         it back through Open vSwitch.
+trex_vm_tput                             T-Rex VM - execute RFC2544 Throughput from T-Rex VM and loop it back
+                                         through Open vSwitch.
+trex_vm2vm_cont                          T-Rex VM2VM - execute RFC2544 Continuous Stream from T-Rex VM and
+                                         loop it back through 2nd VM.
+trex_vm2vm_tput                          T-Rex VM2VM - execute RFC2544 Throughput from T-Rex VM and loop it back
+                                         through 2nd VM.
+
+======================================== ======================================================================================
diff --git a/systems/centos/build_base_machine.sh b/systems/centos/build_base_machine.sh
index a45b0c3d..95f9e211 100755
--- a/systems/centos/build_base_machine.sh
+++ b/systems/centos/build_base_machine.sh
@@ -46,6 +46,7 @@ automake
 pciutils
 cifs-utils
 sysstat
+sshpass
 
 # libs
 libpcap-devel
diff --git a/systems/fedora/24/build_base_machine.sh b/systems/fedora/24/build_base_machine.sh
index 2aceb34f..bbde9b77 100644
--- a/systems/fedora/24/build_base_machine.sh
+++ b/systems/fedora/24/build_base_machine.sh
@@ -48,6 +48,7 @@ pciutils
 cifs-utils
 socat
 sysstat
+sshpass
 
 # install python packages
 python3
diff --git a/systems/fedora/25/build_base_machine.sh b/systems/fedora/25/build_base_machine.sh
index 241b79cb..df8ae620 100644
--- a/systems/fedora/25/build_base_machine.sh
+++ b/systems/fedora/25/build_base_machine.sh
@@ -48,6 +48,7 @@ pciutils
 cifs-utils
 socat
 sysstat
+sshpass
 
 # install python packages
 python3
diff --git a/systems/fedora/26/build_base_machine.sh b/systems/fedora/26/build_base_machine.sh
index b8b65965..84c0695b 100644
--- a/systems/fedora/26/build_base_machine.sh
+++ b/systems/fedora/26/build_base_machine.sh
@@ -48,6 +48,7 @@ pciutils
 cifs-utils
 socat
 sysstat
+sshpass
 
 # install python packages
 python3
diff --git a/systems/rhel/7.2/build_base_machine.sh b/systems/rhel/7.2/build_base_machine.sh
index 858092df..d3f74726 100755
--- a/systems/rhel/7.2/build_base_machine.sh
+++ b/systems/rhel/7.2/build_base_machine.sh
@@ -52,7 +52,9 @@ pkglist=(
  wget\
  numactl\
  numactl-devel\
- libpng-devel
+ libpng-devel\
+ epel-release\
+ sshpass\
 )
 
 # python tools for proper QEMU, DPDK, and OVS make
diff --git a/systems/rhel/7.3/build_base_machine.sh b/systems/rhel/7.3/build_base_machine.sh
index 58953e23..dfa738df 100755
--- a/systems/rhel/7.3/build_base_machine.sh
+++ b/systems/rhel/7.3/build_base_machine.sh
@@ -52,7 +52,9 @@ pkglist=(
  wget\
  numactl\
  numactl-devel\
- libpng-devel
+ libpng-devel\
+ epel-release\
+ sshpass\
 )
 
 # python tools for proper QEMU, DPDK, and OVS make
@@ -102,4 +104,4 @@ rm -f /etc/yum.repos.d/python34.repo
 mkdir -p /dev/hugepages
 
 # prevent ovs vanilla from building from source due to kernel incompatibilities
-sed -i s/'SUBBUILDS = src_vanilla'/'#SUBBUILDS = src_vanilla'/ ../src/Makefile
\ No newline at end of file
+sed -i s/'SUBBUILDS = src_vanilla'/'#SUBBUILDS = src_vanilla'/ ../src/Makefile
diff --git a/systems/ubuntu/14.04/build_base_machine.sh b/systems/ubuntu/14.04/build_base_machine.sh
index 04f4a7e1..a0e6895c 100755
--- a/systems/ubuntu/14.04/build_base_machine.sh
+++ b/systems/ubuntu/14.04/build_base_machine.sh
@@ -62,6 +62,7 @@ socat
 libpixman-1-0
 libpixman-1-dev
 sysstat
+sshpass
 
 # Java runtime environment: Required for Ixia TclClient
 default-jre
diff --git a/systems/ubuntu/build_base_machine.sh b/systems/ubuntu/build_base_machine.sh
index 1b42a790..2f3e7b25 100755
--- a/systems/ubuntu/build_base_machine.sh
+++ b/systems/ubuntu/build_base_machine.sh
@@ -46,6 +46,7 @@ apt-get -y install libglib2.0
 apt-get -y install libfuse-dev
 apt-get -y install libnuma1
 apt-get -y install libnuma-dev
+apt-get -y install sshpass
 
 # Some useful tools you may optionally install
 #apt-get -y install ctags
diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py
index 8e3d44de..a0128f44 100644
--- a/vnfs/qemu/qemu.py
+++ b/vnfs/qemu/qemu.py
@@ -222,7 +222,13 @@ class IVnfQemu(IVnf):
             stdin=proc.stdout)
         proc.wait()
 
-        for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
+        # calculate the number of CPUs in SMP topology specified by GUEST_SMP
+        # e.g. "sockets=2,cores=3", "4", etc.
+        cpu_nr = 1
+        for i in re.findall(r'\d', S.getValue('GUEST_SMP')[self._number]):
+            cpu_nr = cpu_nr * int(i)
+        # pin each GUEST's core to host core based on configured BINDING
+        for cpu in range(0, cpu_nr):
             match = None
             guest_thread_binding = S.getValue('GUEST_THREAD_BINDING')[self._number]
             if guest_thread_binding is None: