tests: L3, L4 and VxLAN tests for OVS & VPP

A set of tests was introduced with focus on L3, L4 and VxLAN performance
of OVS and VPP. New testcases were created for phy2phy network scenario.
In case of PVP and PVVP, only OVS testcases are available.

Notes:
* two sets of OVS P2P testcases were created, one creates unique flow
  for each IP address involved in the test (performance sensitive);
  Second set inserts just one flow with large network mask (tests with
  _mask suffix).
* three different types of VPP P2P L3 testcases were created to
  demonstrate performance impact of multi ARP entries or IP routes.
* VPP multi ARP record based testcases use a set of "workarounds"
  to load a large number of ARP entries. It is not possible to use
  "set ip arp count" syntax, as it doesn't work well for large count
  values (e.g. 60K)
* OVS VxLAN testcases utilize existing OP2P deployment scenario and thus
  it can be used also with GRE and GENEVE tunnel types. Tunnel type
  to be used is defined by test configuration option "Tunnel Type".

JIRA: VSPERF-518

Change-Id: I65adad976f12d8625d918a1996eb42693c511ee1
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com>
Reviewed-by: Trevor Cooper <trevor.cooper@intel.com>
Reviewed-by: Cian Ferriter <cian.ferriter@intel.com>

6 files changed, 1089 insertions, 30 deletions

diff --git a/conf/__init__.py b/conf/__init__.py
index e24111dc..b592165c 100644
--- a/conf/__init__.py
+++ b/conf/__init__.py
@@ -148,14 +148,15 @@ class Settings(object):
 
         :returns: None
         """
-        regex = re.compile("^(?P<digit_part>[0-9]+).*.conf$")
+        regex = re.compile("^(?P<digit_part>[0-9]+)(?P<alfa_part>[a-z]?)_.*.conf$")
 
         def get_prefix(filename):
             """
             Provide a suitable function for sort's key arg
             """
             match_object = regex.search(os.path.basename(filename))
-            return int(match_object.group('digit_part'))
+            return [int(match_object.group('digit_part')),
+                    match_object.group('alfa_part')]
 
         # get full file path to all files & dirs in dir_path
         file_paths = os.listdir(dir_path)
@@ -166,7 +167,7 @@ class Settings(object):
         file_paths = [x for x in file_paths if os.path.isfile(x) and
                       regex.search(os.path.basename(x))]
 
-        # sort ascending on the leading digits
+        # sort ascending on the leading digits and afla (e.g. 03_, 05a_)
         file_paths.sort(key=get_prefix)
 
         # load settings from each file in turn
diff --git a/conf/integration/01_testcases.conf b/conf/integration/01_testcases.conf
index b58fa965..647e39cc 100644
--- a/conf/integration/01_testcases.conf
+++ b/conf/integration/01_testcases.conf
@@ -40,10 +40,19 @@ SUPPORTED_TUNNELING_PROTO = ['vxlan', 'gre', 'geneve']
 #
 
 # P2P macros
-STEP_VSWITCH_P2P_FLOWS_INIT = [
+STEP_VSWITCH_P2P_INIT = [
     ['vswitch', 'add_switch', 'int_br0'],           # STEP 0
     ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 1
     ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 2
+]
+
+STEP_VSWITCH_P2P_FINIT = [
+    ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
+    ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
+    ['vswitch', 'del_switch', 'int_br0'],
+]
+
+STEP_VSWITCH_P2P_FLOWS_INIT = STEP_VSWITCH_P2P_INIT + [
     ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
     ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}],
 ]
@@ -52,27 +61,18 @@ STEP_VSWITCH_P2P_FLOWS_FINIT = [
     ['vswitch', 'dump_flows', 'int_br0'],
     ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[1][1]'}],
     ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[2][1]'}],
-    ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
-    ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
-    ['vswitch', 'del_switch', 'int_br0'],
-]
+] + STEP_VSWITCH_P2P_FINIT
 
 # PVP and PVVP macros
-STEP_VSWITCH_PVP_INIT = [
-    ['vswitch', 'add_switch', 'int_br0'],           # STEP 0
-    ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 1
-    ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 2
+STEP_VSWITCH_PVP_INIT = STEP_VSWITCH_P2P_INIT + [
     ['vswitch', 'add_vport', 'int_br0'],            # STEP 3    vm1 ports
     ['vswitch', 'add_vport', 'int_br0'],            # STEP 4
 ]
 
 STEP_VSWITCH_PVP_FINIT = [
-    ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
-    ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
-    ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'],
+    ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'],  # vm1 ports
     ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'],
-    ['vswitch', 'del_switch', 'int_br0'],
-]
+] + STEP_VSWITCH_P2P_FINIT
 
 STEP_VSWITCH_PVP_FLOWS_INIT = STEP_VSWITCH_PVP_INIT + [
     ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
@@ -988,6 +988,7 @@ INTEGRATION_TESTS = [
     #
     # END of VPP tests used by VERIFY and MERGE jobs by OPNFV Jenkins
     #
+
 ]
 
 # Example of TC definition with exact vSwitch, VNF and TRAFFICGEN values.
diff --git a/conf/integration/01a_testcases_l34_vxlan.conf b/conf/integration/01a_testcases_l34_vxlan.conf
new file mode 100644
index 00000000..17c0d6ff
--- /dev/null
+++ b/conf/integration/01a_testcases_l34_vxlan.conf
@@ -0,0 +1,1007 @@
+# Copyright 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This file introduces a series of scalability testcases prepared for OVS
+# and VPP. Tests are unidirectional and they are focused on L3, L4 and VxLAN
+# switching performance.
+# Following test types are available:
+#   1) unique rule for each IP stream (OVS only)
+#   2) one rule for /8 netmask covering all streams
+#   3) unique ARP entry for each IP stream (VPP only)
+#   4) unique IP route for each IP stream (VPP only)
+#
+
+INTEGRATION_TESTS = INTEGRATION_TESTS + [
+    #
+    # L3 & L4 tests to compare OVS and VPP performance
+    #
+    # Example of execution:
+    #   ./vsperf --test-params "TRAFFIC={'multistream':2000,'traffic_type':'rfc2544_continuous'}" \
+    #       p2p_l3_multi_IP_ovs_mask p2p_l4_multi_PORT_ovs_mask
+    #
+    #   ./vsperf --test-params "TRAFFIC={'multistream':8000,'traffic_type':'rfc2544_throughput'}" \
+    #       p2p_l3_multi_IP_vpp p2p_l4_multi_PORT_vpp
+    {
+        "Name": "p2p_l3_multi_IP_ovs",
+        "Deployment": "clean",
+        "Description": "OVS: P2P L3 multistream with unique flow for each IP stream",
+        "vSwitch" : "OvsDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.1',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'del_flow', 'int_br0'],
+                ['tools', 'exec_python',
+                 'import netaddr;'
+                 'cmds=open("/tmp/ovsofctl_cmds.txt","w");'
+                 '[print("add nw_dst={} idle_timeout=0,dl_type=0x800,'
+                 'in_port=1,action=output:2".format('
+                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i)),file=cmds) '
+                 'for i in range($TRAFFIC["multistream"])];'
+                 'cmds.close()'],
+                ['tools', 'exec_shell', "sudo $TOOLS['ovs-ofctl'] -O OpenFlow13 --bundle add-flows int_br0 /tmp/ovsofctl_cmds.txt"],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'del_flow', 'int_br0'],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "p2p_l3_multi_IP_mask_ovs",
+        "Deployment": "clean",
+        "Description": "OVS: P2P L3 multistream with 1 flow for /8 net mask",
+        "vSwitch" : "OvsDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.1',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'del_flow', 'int_br0'],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0' }],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'dump_flows', 'int_br0'],
+                ['vswitch', 'del_flow', 'int_br0'],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "pvp_l3_multi_IP_mask_ovs",
+        "Deployment": "clean",
+        "Description": "OVS: PVP L3 multistream with 1 flow for /8 net mask",
+        "vSwitch" : "OvsDpdkVhost",
+        "Parameters" : {
+            "GUEST_TESTPMD_FWD_MODE" : ['io'],
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.1',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_PVP_INIT +
+            [
+                ['vswitch', 'del_flow', 'int_br0'],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[4][1]', 'dl_type': '0x800',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
+                ['vnf', 'start'],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'dump_flows', 'int_br0'],
+                ['vnf', 'stop'],
+                ['vswitch', 'del_flow', 'int_br0'],
+            ] +                                                                                                                          STEP_VSWITCH_PVP_FINIT
+    },
+    {
+        "Name": "pvvp_l3_multi_IP_mask_ovs",
+        "Deployment": "clean",
+        "Description": "OVS: PVVP L3 multistream with 1 flow for /8 net mask",
+        "vSwitch" : "OvsDpdkVhost",
+        "Parameters" : {
+            "GUEST_TESTPMD_FWD_MODE" : ['io', 'io'],
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.1',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_PVVP_INIT +
+            [
+                ['vswitch', 'del_flow', 'int_br0'],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[4][1]', 'dl_type': '0x800',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[6][1]', 'dl_type': '0x800',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
+                ['vnf1', 'start'],
+                ['vnf2', 'start'],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'dump_flows', 'int_br0'],
+                ['vnf2', 'stop'],
+                ['vnf1', 'stop'],
+                ['vswitch', 'del_flow', 'int_br0'],
+            ] +
+            STEP_VSWITCH_PVVP_FINIT
+    },
+    {
+        "Name": "p2p_l4_multi_PORT_ovs",
+        "Deployment": "clean",
+        "Description": "OVS: P2P L4 multistream with unique flow for each IP stream",
+        "vSwitch" : "OvsDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'del_flow', 'int_br0'],
+                ['tools', 'exec_python',
+                 'import netaddr;'
+                 'cmds=open("/tmp/ovsofctl_cmds.txt","w");'
+                 '[print("add nw_dst={} idle_timeout=0,dl_type=0x800,nw_proto=17,tp_src={},'
+                 'in_port=1,action=output:2".format('
+                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i),'
+                 '$TRAFFIC["l4"]["srcport"]),file=cmds) '
+                 'for i in range($TRAFFIC["multistream"])];'
+                 'cmds.close()'],
+                ['tools', 'exec_shell', "sudo $TOOLS['ovs-ofctl'] -O OpenFlow13 --bundle "
+                                        "add-flows int_br0 /tmp/ovsofctl_cmds.txt"],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'del_flow', 'int_br0'],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "p2p_l4_multi_PORT_mask_ovs",
+        "Deployment": "clean",
+        "Description": "OVS: P2P L4 multistream with 1 flow for /8 net and port mask",
+        "vSwitch" : "OvsDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'del_flow', 'int_br0'],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
+                 'nw_proto': '17', 'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                 'tp_src': '$TRAFFIC["l4"]["srcport"]',
+                 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'dump_flows', 'int_br0'],
+                ['vswitch', 'del_flow', 'int_br0'],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "pvp_l4_multi_PORT_mask_ovs",
+        "Deployment": "clean",
+        "Description": "OVS: PVP L4 multistream flows for /8 net and port mask",
+        "vSwitch" : "OvsDpdkVhost",
+        "Parameters" : {
+            "GUEST_TESTPMD_FWD_MODE" : ['io'],
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_PVP_INIT +
+            [
+                ['vswitch', 'del_flow', 'int_br0'],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800', 'nw_proto': '17',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
+                  'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[4][1]', 'dl_type': '0x800', 'nw_proto': '17',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
+                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
+                ['vnf', 'start'],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'dump_flows', 'int_br0'],
+                ['vnf', 'stop'],
+                ['vswitch', 'del_flow', 'int_br0'],
+            ] +
+            STEP_VSWITCH_PVP_FINIT
+    },
+    {
+        "Name": "pvvp_l4_multi_PORT_mask_ovs",
+        "Deployment": "clean",
+        "Description": "OVS: PVVP L4 multistream with flows for /8 net and port mask",
+        "vSwitch" : "OvsDpdkVhost",
+        "Parameters" : {
+            "GUEST_TESTPMD_FWD_MODE" : ['io', 'io'],
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_PVVP_INIT +
+            [
+                ['vswitch', 'del_flow', 'int_br0'],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
+                  'nw_proto': '17', 'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
+                  'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[4][1]', 'dl_type': '0x800', 'nw_proto': '17',
+                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
+                  'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}],
+                ['vswitch', 'add_flow', 'int_br0',
+                 {'in_port': '#STEP[6][1]', 'dl_type': '0x800',
+                  'nw_proto': '17', 'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
+                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
+                ['vnf1', 'start'],
+                ['vnf2', 'start'],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'dump_flows', 'int_br0'],
+                ['vnf2', 'stop'],
+                ['vnf1', 'stop'],
+                ['vswitch', 'del_flow', 'int_br0'],
+            ] +
+            STEP_VSWITCH_PVVP_FINIT
+    },
+    {
+        "Name": "p2p_l3_multi_IP_arp_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: P2P L3 multistream with unique ARP entry for each IP stream",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '6.0.0.2',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 5.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.0.1/8']],
+                # count option of "set ip arp" command doesn't work reliably for huge
+                # count numbers, e.g. 100K
+                # it is both more reliale and faster to load batches of sigle
+                # ARP commands, although it also fails time to time
+                # NOTE: batch load of "set ip arp count" commands with lower
+                # count values (e.g. 1000) was also less reliable than using
+                # batches of signle arp entries
+                ['tools', 'exec_python',
+                 'import netaddr;'
+                 'dst_mac_value = netaddr.EUI("00:00:00:00:00:0A").value;'
+                 'cmds=open("/tmp/vppctl_cmds.txt","w");'
+                 '[print("set ip arp #STEP[2][0] {} {}".format('
+                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i),'
+                 'netaddr.EUI(dst_mac_value+i,dialect=netaddr.mac_unix_expanded)),file=cmds) '
+                 'for i in range($TRAFFIC["multistream"])];'
+                 'cmds.close()'],
+                ['tools', 'exec_shell', "rm -rf /tmp/vppctl_cmds_split*; split -l 1000 "
+                                        "/tmp/vppctl_cmds.txt /tmp/vppctl_cmds_split"],
+                ['tools', 'exec_shell', "for a in /tmp/vppctl_cmds_split* ; do echo $a ; "
+                                        "sudo $TOOLS['vppctl'] exec $a ; sleep 2 ; done"],
+                ['vswitch', 'run_vppctl', ['show ip fib summary']],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "p2p_l3_multi_IP_mask_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: P2P L3 multistream with 1 route for /8 net mask",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '5.0.0.1',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.3.1/24']],
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.3.2 00:00:00:00:00:0A']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 via 6.0.3.2']],
+                ['vswitch', 'run_vppctl', ['show ip fib']],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "p2p_l3_multi_IP_routes_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: P2P L3 multistream with unique route for each IP stream",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '5.0.0.1',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.3.1/24']],
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.3.2 00:00:00:00:00:0A']],
+                # insertion of huge number of IP routes doesn't cause issues
+                # seen with ARP entries at p2p_l3_multi_IP_vpp testcase
+                ['vswitch', 'run_vppctl', ['ip route add count $TRAFFIC["multistream"] '
+                                           '$TRAFFIC["l3"]["dstip"]/32 via 6.0.3.2']],
+                ['vswitch', 'run_vppctl', ['show ip fib summary']],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "pvp_l3_multi_IP_mask_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: PVP L3 multistream with route for /8 netmask",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "GUEST_TESTPMD_FWD_MODE" : ['io'],
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '5.0.0.1',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_PVP_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set interface mac address #STEP[4][0] 00:00:00:00:00:11']],
+                # two separate tables are used, so the same IP network can be routed
+                # via different DST IPs to reach VM and trafficgen via 2nd phy NIC
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[1][0] 6']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[3][0] 6']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.1.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[3][0] 6.0.3.1/24']],
+                # route traffic to VM; Set DST MAC to MAC of 2nd (output) vhost user interface
+                # of VM, so traffic is accepted by VPP in next table
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[3][0] 6.0.3.2 00:00:00:00:00:11 fib-id 6']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 6 via 6.0.3.2']],
+
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[2][0] 7']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[4][0] 7']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[4][0] 6.0.4.1/24']],
+                # route traffic via 2nd phy NIC to reach trafficgen
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.2.2 00:00:00:00:00:0A fib-id 7']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 7 via 6.0.2.2']],
+
+                ['vswitch', 'run_vppctl', ['show ip fib']],
+                ['vnf', 'start'],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+                ['vnf', 'stop'],
+            ] +
+            STEP_VSWITCH_PVP_FINIT
+    },
+    {
+        "Name": "pvvp_l3_multi_IP_mask_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: PVVP L3 multistream with route for /8 netmask",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "GUEST_TESTPMD_FWD_MODE" : ['io', 'io'],
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '5.0.0.1',
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_PVVP_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set interface mac address #STEP[4][0] 00:00:00:00:00:11']],
+                ['vswitch', 'run_vppctl', ['set interface mac address #STEP[6][0] 00:00:00:00:00:12']],
+                # three separate tables are used, so the same IP network can be routed
+                # via different DST IPs to reach both VMs and trafficgen via 2nd phy NIC
+                # 1st table
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[1][0] 6']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[3][0] 6']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.1.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[3][0] 6.0.3.1/24']],
+                # route traffic to 1st VM; Set DST MAC to MAC of 2nd (output) vhost user interface
+                # of VM1, so traffic is accepted by VPP in next table
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[3][0] 6.0.3.2 00:00:00:00:00:11 fib-id 6']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 6 via 6.0.3.2']],
+                # 2nd table
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[4][0] 7']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[5][0] 7']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[4][0] 6.0.4.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[5][0] 6.0.5.1/24']],
+                # route traffic to 2nd VM; Set DST MAC to MAC of 2nd (output) vhost user interfacei
+                # of VM2, so traffic is accepted by VPP in next table
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[5][0] 6.0.5.2 00:00:00:00:00:12 fib-id 7']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 7 via 6.0.5.2']],
+                # 3rd table
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[2][0] 8']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[6][0] 8']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[6][0] 6.0.6.1/24']],
+                # route traffic via 2nd phy NIC to reach trafficgen
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.2.2 00:00:00:00:00:0A fib-id 8']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 8 via 6.0.2.2']],
+                ['vswitch', 'run_vppctl', ['show ip fib']],
+                ['vnf1', 'start'],
+                ['vnf2', 'start'],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+                ['vnf1', 'stop'],
+                ['vnf2', 'stop'],
+            ] +
+            STEP_VSWITCH_PVVP_FINIT
+    },
+    {
+        "Name": "p2p_l4_multi_PORT_arp_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: P2P L4 multistream with unique ARP entry for each IP stream and port check",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'srcip': '6.0.2.2',
+                    'dstip': '5.0.0.2',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 5.0.0.1/8']],
+                ['tools', 'exec_python',
+                 'import netaddr;'
+                 'dst_mac_value = netaddr.EUI("00:00:00:00:00:0A").value;'
+                 'cmds=open("/tmp/vppctl_cmds.txt","w");'
+                 '[print("set ip arp #STEP[2][0] {} {}".format('
+                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i),'
+                 'netaddr.EUI(dst_mac_value+i,dialect=netaddr.mac_unix_expanded)),file=cmds) '
+                 'for i in range($TRAFFIC["multistream"])];'
+                 'cmds.close()'],
+                ['vswitch', 'run_vppctl',
+                 ['set ip source-and-port-range-check vrf 7 $TRAFFIC["l3"]["srcip"]/24 '
+                  'port $TRAFFIC["l4"]["dstport"]']],
+                ['vswitch', 'run_vppctl',
+                 ['set interface ip source-and-port-range-check #STEP[1][0] udp-out-vrf 7']],
+                ['tools', 'exec_shell', "rm -rf /tmp/vppctl_cmds_split*; split -l 1000 "
+                                        "/tmp/vppctl_cmds.txt /tmp/vppctl_cmds_split"],
+                ['tools', 'exec_shell', "for a in /tmp/vppctl_cmds_split* ; do echo $a ; "
+                                        "sudo $TOOLS['vppctl'] exec $a ; sleep 2 ; done"],
+                ['vswitch', 'run_vppctl', ['show ip fib summary']],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "p2p_l4_multi_PORT_mask_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: P2P L4 multistream with 1 route for /8 net mask and port check",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'srcip': '6.0.2.2',
+                    'dstip': '5.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']], # STEP 3
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.3.1/24']], # STEP 4
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.3.2 00:00:00:00:00:0A']], # STEP 5
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 via 6.0.3.2']],
+                ['vswitch', 'run_vppctl', ['show ip fib']],
+                ['vswitch', 'run_vppctl',
+                 ['set ip source-and-port-range-check vrf 7 $TRAFFIC["l3"]["srcip"]/24 '
+                  'port $TRAFFIC["l4"]["dstport"]']],
+                ['vswitch', 'run_vppctl',
+                 ['set interface ip source-and-port-range-check #STEP[1][0] udp-out-vrf 7']],
+                ['vswitch', 'run_vppctl', ['show ip fib']],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "p2p_l4_multi_PORT_routes_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: P2P L4 multistream with unique route for each IP stream and port check",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'srcip': '6.0.2.2',
+                    'dstip': '5.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.3.1/24']],
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.3.2 00:00:00:00:00:0A']],
+                # insertion of huge number of IP routes doesn't cause issues
+                # seen with ARP entries at p2p_l3_multi_IP_vpp testcase
+                ['vswitch', 'run_vppctl',
+                 ['ip route add count $TRAFFIC["multistream"] $TRAFFIC["l3"]["dstip"]/32 via 6.0.3.2']],
+                ['vswitch', 'run_vppctl',
+                 ['set ip source-and-port-range-check vrf 7 $TRAFFIC["l3"]["srcip"]/24 '
+                  'port $TRAFFIC["l4"]["dstport"]']],
+                ['vswitch', 'run_vppctl',
+                 ['set interface ip source-and-port-range-check #STEP[1][0] udp-out-vrf 7']],
+                ['vswitch', 'run_vppctl', ['show ip fib summary']],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "pvp_l4_multi_PORT_mask_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: PVP L4 multistream with route for /8 net and port mask",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "GUEST_TESTPMD_FWD_MODE" : ['io'],
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'srcip': '4.0.0.1',
+                    'dstip': '5.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_PVP_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set int mac address #STEP[4][0] 00:00:00:00:00:11']],
+                # create table for port check
+                ['vswitch', 'run_vppctl', ['set ip source-and-port-range-check vrf 5 '
+                                           '$TRAFFIC["l3"]["srcip"]/24 '
+                  'port $TRAFFIC["l4"]["dstport"]']],
+                # two separate tables are used, so the same IP network can be routed
+                # via different DST IPs to reach VM and trafficgen via 2nd phy NIC
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[1][0] 6']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[3][0] 6']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.1.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[3][0] 6.0.3.1/24']],
+                # enforce port check
+                ['vswitch', 'run_vppctl', ['set int ip source-and-port-range-check #STEP[1][0] '
+                                           'udp-out-vrf 5']],
+                # route traffic to VM; Set DST MAC to MAC of 2nd (output) vhost user interface
+                # of VM, so traffic is accepted by VPP in next table
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[3][0] 6.0.3.2 00:00:00:00:00:11 fib-id 6']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 6 via 6.0.3.2']],
+
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[2][0] 7']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[4][0] 7']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[4][0] 6.0.4.1/24']],
+                # route traffic via 2nd phy NIC to reach trafficgen
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.2.2 00:00:00:00:00:0A fib-id 7']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 7 via 6.0.2.2']],
+
+                ['vswitch', 'run_vppctl', ['show ip fib']],
+                ['vnf', 'start'],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+                ['vnf', 'stop'],
+            ] +
+            STEP_VSWITCH_PVP_FINIT
+    },
+    {
+        "Name": "pvvp_l4_multi_PORT_mask_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: PVVP L4 multistream with route for /8 net and port mask",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "GUEST_TESTPMD_FWD_MODE" : ['io', 'io'],
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'srcip': '4.0.0.1',
+                    'dstip': '5.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                    'srcport': 7,
+                    'dstport': 8,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_PVVP_INIT +
+            [
+                ['vswitch', 'run_vppctl', ['set int mac address #STEP[4][0] 00:00:00:00:00:11']],
+                ['vswitch', 'run_vppctl', ['set int mac address #STEP[6][0] 00:00:00:00:00:12']],
+                # create table for port check
+                ['vswitch', 'run_vppctl', ['set ip source-and-port-range-check vrf 5 '
+                                           '$TRAFFIC["l3"]["srcip"]/24 port $TRAFFIC["l4"]["dstport"]']],
+                # three separate tables are used, so the same IP network can be routed
+                # via different DST IPs to reach both VMs and trafficgen via 2nd phy NIC
+                # 1st table
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[1][0] 6']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[3][0] 6']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.1.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[3][0] 6.0.3.1/24']],
+                # enforce port check
+                ['vswitch', 'run_vppctl', ['set int ip source-and-port-range-check #STEP[1][0] udp-out-vrf 5']],
+                # route traffic to 1st VM; Set DST MAC to MAC of 2nd (output) vhost user interface
+                # of VM1, so traffic is accepted by VPP in next table
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[3][0] 6.0.3.2 00:00:00:00:00:11 fib-id 6']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 6 via 6.0.3.2']],
+                # 2nd table
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[4][0] 7']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[5][0] 7']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[4][0] 6.0.4.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[5][0] 6.0.5.1/24']],
+                # route traffic to 2nd VM; Set DST MAC to MAC of 2nd (output) vhost user interfacei
+                # of VM2, so traffic is accepted by VPP in next table
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[5][0] 6.0.5.2 00:00:00:00:00:12 fib-id 7']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 7 via 6.0.5.2']],
+                # 3rd table
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[2][0] 8']],
+                ['vswitch', 'run_vppctl', ['set int ip table #STEP[6][0] 8']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.2.1/24']],
+                ['vswitch', 'run_vppctl', ['set int ip address #STEP[6][0] 6.0.6.1/24']],
+                # route traffic via 2nd phy NIC to reach trafficgen
+                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.2.2 00:00:00:00:00:0A fib-id 8']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 8 via 6.0.2.2']],
+                ['vswitch', 'run_vppctl', ['show ip fib']],
+                ['vnf1', 'start'],
+                ['vnf2', 'start'],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+                ['vnf1', 'stop'],
+                ['vnf2', 'stop'],
+            ] +
+            STEP_VSWITCH_PVVP_FINIT
+    },
+    #
+    # End of L3 & L4 tests to compare OVS and VPP performance
+    #
+    #
+    # VxLAN tests to compare OVS and VPP performance
+    #
+    # Example of execution:
+    #   ./vsperf --test-params "TRAFFIC={'multistream':2000,'traffic_type':'rfc2544_continuous'}" \
+    #       vxlan_multi_IP_vpp
+    {
+        "Name": "vxlan_multi_IP_mask_ovs",
+        "Deployment": "op2p",
+        "Tunnel Type": "vxlan",
+        "Tunnel Operation": "encapsulation",
+        "Description": "OVS: VxLAN L3 multistream",
+        "Parameters": {
+            "TRAFFICGEN_IXNET_TCL_SCRIPT" : "ixnetrfc2544v2.tcl",
+            "TRAFFICGEN_PORT1_MAC" : '00:00:00:00:00:01',
+            "TRAFFICGEN_PORT2_MAC" : '#PARAM(NICS[0]["mac"])',
+            "TRAFFICGEN_PORT1_IP"  : '9.0.0.2',
+            "TRAFFICGEN_PORT2_IP"  : '10.0.0.2',
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+            },
+        },
+        "TestSteps": [
+            ['vswitch', 'del_flow', 'br0', {'in_port': '1'}],
+            ['vswitch', 'add_flow', 'br0',
+             {'in_port': '1', 'dl_type': '0x800', 'nw_proto': '17',
+              'nw_dst': '$TRAFFICGEN_PORT2_IP/8', 'actions': ['output:2'],
+              'idle_timeout': '0'}],
+            ['vswitch', 'dump_flows', 'br0'],
+            ['vswitch', 'dump_flows', 'br-ext'],
+        ],
+    },
+    {
+        "Name": "vxlan_multi_IP_arp_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: VxLAN L3 multistream with unique ARP entry for each IP stream",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '10.0.0.2',
+                },
+                'l4': {
+                    'enabled': True,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'run_vppctl',
+                 ['set int ip address #STEP[1][0] 9.0.0.1/16']],
+                ['vswitch', 'run_vppctl',
+                 ['set int ip address #STEP[2][0] 11.0.0.1/16']],
+                ['vswitch', 'run_vppctl',
+                 ['create vxlan tunnel src 11.0.0.1 dst 11.0.0.2 vni 1']],
+                ['vswitch', 'run_vppctl',
+                 ['set int state vxlan_tunnel0 up']],
+                ['vswitch', 'run_vppctl',
+                 ['set int ip address vxlan_tunnel0 10.0.0.1/8']],
+                ['vswitch', 'run_vppctl',
+                 ['set ip arp #STEP[2][0] 11.0.0.2 00:00:00:00:00:00']],
+                ['tools', 'exec_python',
+                 'import netaddr;'
+                 'dst_mac_value = netaddr.EUI("00:00:00:00:00:0A").value;'
+                 'cmds=open("/tmp/vppctl_cmds.txt","w");'
+                 '[print("set ip arp vxlan_tunnel0 {} {}".format('
+                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i),'
+                 'netaddr.EUI(dst_mac_value+i,dialect=netaddr.mac_unix_expanded)),file=cmds) '
+                 'for i in range($TRAFFIC["multistream"])];'
+                 'cmds.close()'],
+                ['tools', 'exec_shell',
+                 "rm -rf /tmp/vppctl_cmds_split*;"
+                 "split -l 1000 /tmp/vppctl_cmds.txt /tmp/vppctl_cmds_split"],
+                ['tools', 'exec_shell',
+                 "for a in /tmp/vppctl_cmds_split* ; do "
+                 "echo $a ; sudo $TOOLS['vppctl'] exec $a ; sleep 2 ; done"],
+                ['vswitch', 'run_vppctl', ['show ip fib summary']],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    {
+        "Name": "vxlan_multi_IP_mask_vpp",
+        "Deployment": "clean",
+        "Description": "VPP: VxLAN L3 multistream with 1 route for /8 netmask",
+        "vSwitch" : "VppDpdkVhost",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'bidir' : 'False',
+                "stream_type" : "L3",
+                'l2': {
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+                'l3': {
+                    'enabled': True,
+                    'proto': 'udp',
+                    'dstip': '8.0.0.1',
+                },
+                'l4': {
+                    'enabled': True,
+                },
+            },
+        },
+        "TestSteps":
+            STEP_VSWITCH_P2P_INIT +
+            [
+                ['vswitch', 'run_vppctl',
+                 ['set int ip address #STEP[1][0] 9.0.0.1/16']],
+                ['vswitch', 'run_vppctl',
+                 ['set int ip address #STEP[2][0] 11.0.0.1/16']],
+                ['vswitch', 'run_vppctl',
+                 ['create vxlan tunnel src 11.0.0.1 dst 11.0.0.2 vni 1']],
+                ['vswitch', 'run_vppctl',
+                 ['set int state vxlan_tunnel0 up']],
+                ['vswitch', 'run_vppctl',
+                 ['set int ip address vxlan_tunnel0 10.0.0.1/24']],
+                ['vswitch', 'run_vppctl',
+                 ['set ip arp #STEP[2][0] 11.0.0.2 00:00:00:00:00:00']],
+                ['vswitch', 'run_vppctl',
+                 ['set ip arp vxlan_tunnel0 10.0.0.2 00:00:00:00:00:0A']],
+                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 via 10.0.0.2']],
+                ['vswitch', 'run_vppctl', ['show ip fib']],
+                ['trafficgen', 'send_traffic', {}],
+                ['vswitch', 'run_vppctl', ['show interfaces']],
+            ] +
+            STEP_VSWITCH_P2P_FINIT
+    },
+    #
+    # End of VxLAN tests to compare OVS and VPP performance
+    #
+]
diff --git a/docs/testing/developer/design/vswitchperf_design.rst b/docs/testing/developer/design/vswitchperf_design.rst
index a5a99c8a..8111b513 100644
--- a/docs/testing/developer/design/vswitchperf_design.rst
+++ b/docs/testing/developer/design/vswitchperf_design.rst
@@ -94,11 +94,13 @@ Configuration Procedure and Precedence
 --------------------------------------
 
 Configuration files follow a strict naming convention that allows them to be
-processed in a specific order. All the .conf files are named ``NN_name.conf``,
-where NN is a decimal number. The files are processed in order from 00_name.conf
-to 99_name.conf so that if the name setting is given in both a lower and higher
-numbered conf file then the higher numbered file is the effective setting as it
-is processed after the setting in the lower numbered file.
+processed in a specific order. All the .conf files are named ``NNx_name.conf``,
+where ``NN`` is a decimal number and ``x`` is an optional alphabetical suffix.
+The files are processed in order from ``00_name.conf`` to ``99_name.conf``
+(and from ``00a_name`` to ``00z_name``), so that if the name setting is given
+in both a lower and higher numbered conf file then the higher numbered file
+is the effective setting as it is processed after the setting in the lower
+numbered file.
 
 The values in the file specified by ``--conf-file`` takes precedence over all
 the other configuration files and does not have to follow the naming
diff --git a/docs/testing/user/userguide/teststeps.rst b/docs/testing/user/userguide/teststeps.rst
index 71f19714..5349d2ea 100644
--- a/docs/testing/user/userguide/teststeps.rst
+++ b/docs/testing/user/userguide/teststeps.rst
@@ -143,6 +143,43 @@ of supported objects and their most common functions follows:
 
             ['settings', 'setValue', 'GUEST_USERNAME', ['root']]
 
+        It is possible and more convenient to access any VSPERF configuration option directly
+        via ``$NAME`` notation. Option evaluation is done during runtime and vsperf will
+        automatically translate it to the appropriate call of ``settings.getValue``.
+        If the referred parameter does not exist, then vsperf will keep ``$NAME``
+        string untouched and it will continue with testcase execution. The reason is to
+        avoid test execution failure in case that ``$`` sign has been used from different
+        reason than vsperf parameter evaluation.
+
+        **NOTE:** It is recommended to use ``${NAME}`` notation for any shell parameters
+        used within ``Exec_Shell`` call to avoid a clash with configuration parameter
+        evaluation.
+
+        **NOTE:** It is possible to refer to vsperf parameter value by ``#PARAM()`` macro
+        (see :ref:`overriding-parameters-documentation`. However ``#PARAM()`` macro is
+        evaluated at the beginning of vsperf execution and it will not reflect any changes
+        made to the vsperf configuration during runtime. On the other hand ``$NAME``
+        notation is evaluated during test execution and thus it contains any modifications
+        to the configuration parameter made by vsperf (e.g. ``TOOLS`` and ``NICS``
+        dictionaries) or by testcase definition (e.g. ``TRAFFIC`` dictionary).
+
+        Examples:
+
+        .. code-block:: python
+
+            ['tools', 'exec_shell', "$TOOLS['ovs-vsctl'] show"]
+
+            ['settings', 'setValue', 'TRAFFICGEN_IXIA_PORT2', '$TRAFFICGEN_IXIA_PORT1'],
+
+            ['vswitch', 'add_flow', 'int_br0',
+             {'in_port': '#STEP[1][1]',
+              'dl_type': '0x800',
+              'nw_proto': '17',
+              'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
+              'actions': ['output:#STEP[2][1]']
+             }
+            ]
+
     * ``namespace`` - creates or modifies network namespaces
 
       List of supported functions:
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 17bce38e..a213bbe8 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -681,8 +681,7 @@ class TestCase(object):
             if self._step_vnf_list[vnf]:
                 self._step_vnf_list[vnf].stop()
 
-    @staticmethod
-    def step_eval_param(param, STEP):
+    def step_eval_param(self, param, STEP):
         # pylint: disable=invalid-name
         """ Helper function for #STEP macro evaluation
         """
@@ -694,28 +693,40 @@ class TestCase(object):
                     # pylint: disable=eval-used
                     tmp_val = str(eval(macro[1:]))
                     param = param.replace(macro, tmp_val)
+
+            # evaluate references to vsperf configuration options
+            macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
+            if macros:
+                for macro in macros:
+                    # pylint: disable=eval-used
+                    try:
+                        tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
+                        param = param.replace('${}'.format(macro[0]), tmp_val)
+                    # ignore that required option can't be evaluated
+                    except (IndexError, KeyError, AttributeError):
+                        self._logger.debug("Skipping %s as it isn't a configuration "
+                                           "parameter.", '${}'.format(macro[0]))
             return param
         elif isinstance(param, list) or isinstance(param, tuple):
             tmp_list = []
             for item in param:
-                tmp_list.append(TestCase.step_eval_param(item, STEP))
+                tmp_list.append(self.step_eval_param(item, STEP))
             return tmp_list
         elif isinstance(param, dict):
             tmp_dict = {}
             for (key, value) in param.items():
-                tmp_dict[key] = TestCase.step_eval_param(value, STEP)
+                tmp_dict[key] = self.step_eval_param(value, STEP)
             return tmp_dict
         else:
             return param
 
-    @staticmethod
-    def step_eval_params(params, step_result):
+    def step_eval_params(self, params, step_result):
         """ Evaluates referrences to results from previous steps
         """
         eval_params = []
         # evaluate all parameters if needed
         for param in params:
-            eval_params.append(TestCase.step_eval_param(param, step_result))
+            eval_params.append(self.step_eval_param(param, step_result))
         return eval_params
 
     def step_run(self):
@@ -788,7 +799,7 @@ class TestCase(object):
             try:
                 # eval parameters, but use only valid step_results
                 # to support negative indexes
-                step_params = TestCase.step_eval_params(step[2:], self._step_result[:i])
+                step_params = self.step_eval_params(step[2:], self._step_result[:i])
                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
                 self._logger.debug("Step %s '%s' start", i, step_log)
                 self._step_result[i] = test_method(*step_params)