# Copyright 2017 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# This file introduces a series of scalability testcases prepared for OVS
# and VPP. Tests are unidirectional and they are focused on L3, L4 and VxLAN
# switching performance.
# Following test types are available:
#   1) unique rule for each IP stream (OVS only)
#   2) one rule for /8 netmask covering all streams
#   3) unique ARP entry for each IP stream (VPP only)
#   4) unique IP route for each IP stream (VPP only)
#

INTEGRATION_TESTS = INTEGRATION_TESTS + [
    #
    # L3 & L4 tests to compare OVS and VPP performance
    #
    # Example of execution:
    #   ./vsperf --test-params "TRAFFIC={'multistream':2000,'traffic_type':'rfc2544_continuous'}" \
    #       p2p_l3_multi_IP_ovs_mask p2p_l4_multi_PORT_ovs_mask
    #
    #   ./vsperf --test-params "TRAFFIC={'multistream':8000,'traffic_type':'rfc2544_throughput'}" \
    #       p2p_l3_multi_IP_vpp p2p_l4_multi_PORT_vpp
    {
        "Name": "p2p_l3_multi_IP_ovs",
        "Deployment": "clean",
        "Description": "OVS: P2P L3 multistream with unique flow for each IP stream",
        "vSwitch" : "OvsDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.1',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'del_flow', 'int_br0'],
                ['tools', 'exec_python',
                 'import netaddr;'
                 'cmds=open("/tmp/ovsofctl_cmds.txt","w");'
                 '[print("add nw_dst={} idle_timeout=0,dl_type=0x800,'
                 'in_port=1,action=output:2".format('
                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i)),file=cmds) '
                 'for i in range($TRAFFIC["multistream"])];'
                 'cmds.close()'],
                ['tools', 'exec_shell', "sudo $TOOLS['ovs-ofctl'] -O OpenFlow13 --bundle add-flows int_br0 /tmp/ovsofctl_cmds.txt"],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'del_flow', 'int_br0'],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "p2p_l3_multi_IP_mask_ovs",
        "Deployment": "clean",
        "Description": "OVS: P2P L3 multistream with 1 flow for /8 net mask",
        "vSwitch" : "OvsDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.1',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'del_flow', 'int_br0'],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0' }],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'dump_flows', 'int_br0'],
                ['vswitch', 'del_flow', 'int_br0'],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "pvp_l3_multi_IP_mask_ovs",
        "Deployment": "clean",
        "Description": "OVS: PVP L3 multistream with 1 flow for /8 net mask",
        "vSwitch" : "OvsDpdkVhost",
        "Parameters" : {
            "GUEST_TESTPMD_FWD_MODE" : ['io'],
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.1',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_PVP_INIT +
            [
                ['vswitch', 'del_flow', 'int_br0'],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[4][1]', 'dl_type': '0x800',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
                ['vnf', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'dump_flows', 'int_br0'],
                ['vnf', 'stop'],
                ['vswitch', 'del_flow', 'int_br0'],
            ] +                                                                                                                          STEP_VSWITCH_PVP_FINIT
    },
    {
        "Name": "pvvp_l3_multi_IP_mask_ovs",
        "Deployment": "clean",
        "Description": "OVS: PVVP L3 multistream with 1 flow for /8 net mask",
        "vSwitch" : "OvsDpdkVhost",
        "Parameters" : {
            "GUEST_TESTPMD_FWD_MODE" : ['io', 'io'],
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.1',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_PVVP_INIT +
            [
                ['vswitch', 'del_flow', 'int_br0'],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[4][1]', 'dl_type': '0x800',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[6][1]', 'dl_type': '0x800',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
                ['vnf1', 'start'],
                ['vnf2', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'dump_flows', 'int_br0'],
                ['vnf2', 'stop'],
                ['vnf1', 'stop'],
                ['vswitch', 'del_flow', 'int_br0'],
            ] +
            STEP_VSWITCH_PVVP_FINIT
    },
    {
        "Name": "p2p_l4_multi_PORT_ovs",
        "Deployment": "clean",
        "Description": "OVS: P2P L4 multistream with unique flow for each IP stream",
        "vSwitch" : "OvsDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.1',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'del_flow', 'int_br0'],
                ['tools', 'exec_python',
                 'import netaddr;'
                 'cmds=open("/tmp/ovsofctl_cmds.txt","w");'
                 '[print("add nw_dst={} idle_timeout=0,dl_type=0x800,nw_proto=17,tp_src={},'
                 'in_port=1,action=output:2".format('
                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i),'
                 '$TRAFFIC["l4"]["srcport"]),file=cmds) '
                 'for i in range($TRAFFIC["multistream"])];'
                 'cmds.close()'],
                ['tools', 'exec_shell', "sudo $TOOLS['ovs-ofctl'] -O OpenFlow13 --bundle "
                                        "add-flows int_br0 /tmp/ovsofctl_cmds.txt"],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'del_flow', 'int_br0'],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "p2p_l4_multi_PORT_mask_ovs",
        "Deployment": "clean",
        "Description": "OVS: P2P L4 multistream with 1 flow for /8 net and port mask",
        "vSwitch" : "OvsDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.1',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'del_flow', 'int_br0'],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
                 'nw_proto': '17', 'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                 'tp_src': '$TRAFFIC["l4"]["srcport"]',
                 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'dump_flows', 'int_br0'],
                ['vswitch', 'del_flow', 'int_br0'],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "pvp_l4_multi_PORT_mask_ovs",
        "Deployment": "clean",
        "Description": "OVS: PVP L4 multistream flows for /8 net and port mask",
        "vSwitch" : "OvsDpdkVhost",
        "Parameters" : {
            "GUEST_TESTPMD_FWD_MODE" : ['io'],
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.1',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_PVP_INIT +
            [
                ['vswitch', 'del_flow', 'int_br0'],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800', 'nw_proto': '17',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
                  'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[4][1]', 'dl_type': '0x800', 'nw_proto': '17',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
                ['vnf', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'dump_flows', 'int_br0'],
                ['vnf', 'stop'],
                ['vswitch', 'del_flow', 'int_br0'],
            ] +
            STEP_VSWITCH_PVP_FINIT
    },
    {
        "Name": "pvvp_l4_multi_PORT_mask_ovs",
        "Deployment": "clean",
        "Description": "OVS: PVVP L4 multistream with flows for /8 net and port mask",
        "vSwitch" : "OvsDpdkVhost",
        "Parameters" : {
            "GUEST_TESTPMD_FWD_MODE" : ['io', 'io'],
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.1',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_PVVP_INIT +
            [
                ['vswitch', 'del_flow', 'int_br0'],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[1][1]', 'dl_type': '0x800',
                  'nw_proto': '17', 'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
                  'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[4][1]', 'dl_type': '0x800', 'nw_proto': '17',
                  'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
                  'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}],
                ['vswitch', 'add_flow', 'int_br0',
                 {'in_port': '#STEP[6][1]', 'dl_type': '0x800',
                  'nw_proto': '17', 'nw_dst': '$TRAFFIC["l3"]["dstip"]/8',
                  'tp_src': '$TRAFFIC["l4"]["srcport"]',
                  'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
                ['vnf1', 'start'],
                ['vnf2', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'dump_flows', 'int_br0'],
                ['vnf2', 'stop'],
                ['vnf1', 'stop'],
                ['vswitch', 'del_flow', 'int_br0'],
            ] +
            STEP_VSWITCH_PVVP_FINIT
    },
    {
        "Name": "p2p_l3_multi_IP_arp_vpp",
        "Deployment": "clean",
        "Description": "VPP: P2P L3 multistream with unique ARP entry for each IP stream",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '6.0.0.2',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 5.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.0.1/8']],
                # count option of "set ip arp" command doesn't work reliably for huge
                # count numbers, e.g. 100K
                # it is both more reliale and faster to load batches of sigle
                # ARP commands, although it also fails time to time
                # NOTE: batch load of "set ip arp count" commands with lower
                # count values (e.g. 1000) was also less reliable than using
                # batches of signle arp entries
                ['tools', 'exec_python',
                 'import netaddr;'
                 'dst_mac_value = netaddr.EUI("00:00:00:00:00:0A").value;'
                 'cmds=open("/tmp/vppctl_cmds.txt","w");'
                 '[print("set ip arp #STEP[2][0] {} {}".format('
                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i),'
                 'netaddr.EUI(dst_mac_value+i,dialect=netaddr.mac_unix_expanded)),file=cmds) '
                 'for i in range($TRAFFIC["multistream"])];'
                 'cmds.close()'],
                ['tools', 'exec_shell', "rm -rf /tmp/vppctl_cmds_split*; split -l 1000 "
                                        "/tmp/vppctl_cmds.txt /tmp/vppctl_cmds_split"],
                ['tools', 'exec_shell', "for a in /tmp/vppctl_cmds_split* ; do echo $a ; "
                                        "sudo $TOOLS['vppctl'] exec $a ; sleep 2 ; done"],
                ['vswitch', 'run_vppctl', ['show ip fib summary']],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "p2p_l3_multi_IP_mask_vpp",
        "Deployment": "clean",
        "Description": "VPP: P2P L3 multistream with 1 route for /8 net mask",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '5.0.0.1',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.3.1/24']],
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.3.2 00:00:00:00:00:0A']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 via 6.0.3.2']],
                ['vswitch', 'run_vppctl', ['show ip fib']],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "p2p_l3_multi_IP_routes_vpp",
        "Deployment": "clean",
        "Description": "VPP: P2P L3 multistream with unique route for each IP stream",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '5.0.0.1',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.3.1/24']],
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.3.2 00:00:00:00:00:0A']],
                # insertion of huge number of IP routes doesn't cause issues
                # seen with ARP entries at p2p_l3_multi_IP_vpp testcase
                ['vswitch', 'run_vppctl', ['ip route add count $TRAFFIC["multistream"] '
                                           '$TRAFFIC["l3"]["dstip"]/32 via 6.0.3.2']],
                ['vswitch', 'run_vppctl', ['show ip fib summary']],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "pvp_l3_multi_IP_mask_vpp",
        "Deployment": "clean",
        "Description": "VPP: PVP L3 multistream with route for /8 netmask",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "GUEST_TESTPMD_FWD_MODE" : ['io'],
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '5.0.0.1',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_PVP_INIT +
            [
                ['vswitch', 'run_vppctl', ['set interface mac address #STEP[4][0] 00:00:00:00:00:11']],
                # two separate tables are used, so the same IP network can be routed
                # via different DST IPs to reach VM and trafficgen via 2nd phy NIC
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[1][0] 6']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[3][0] 6']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.1.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[3][0] 6.0.3.1/24']],
                # route traffic to VM; Set DST MAC to MAC of 2nd (output) vhost user interface
                # of VM, so traffic is accepted by VPP in next table
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[3][0] 6.0.3.2 00:00:00:00:00:11 fib-id 6']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 6 via 6.0.3.2']],

                ['vswitch', 'run_vppctl', ['set int ip table #STEP[2][0] 7']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[4][0] 7']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[4][0] 6.0.4.1/24']],
                # route traffic via 2nd phy NIC to reach trafficgen
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.2.2 00:00:00:00:00:0A fib-id 7']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 7 via 6.0.2.2']],

                ['vswitch', 'run_vppctl', ['show ip fib']],
                ['vnf', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
                ['vnf', 'stop'],
            ] +
            STEP_VSWITCH_PVP_FINIT
    },
    {
        "Name": "pvvp_l3_multi_IP_mask_vpp",
        "Deployment": "clean",
        "Description": "VPP: PVVP L3 multistream with route for /8 netmask",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "GUEST_TESTPMD_FWD_MODE" : ['io', 'io'],
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '5.0.0.1',
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_PVVP_INIT +
            [
                ['vswitch', 'run_vppctl', ['set interface mac address #STEP[4][0] 00:00:00:00:00:11']],
                ['vswitch', 'run_vppctl', ['set interface mac address #STEP[6][0] 00:00:00:00:00:12']],
                # three separate tables are used, so the same IP network can be routed
                # via different DST IPs to reach both VMs and trafficgen via 2nd phy NIC
                # 1st table
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[1][0] 6']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[3][0] 6']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.1.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[3][0] 6.0.3.1/24']],
                # route traffic to 1st VM; Set DST MAC to MAC of 2nd (output) vhost user interface
                # of VM1, so traffic is accepted by VPP in next table
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[3][0] 6.0.3.2 00:00:00:00:00:11 fib-id 6']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 6 via 6.0.3.2']],
                # 2nd table
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[4][0] 7']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[5][0] 7']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[4][0] 6.0.4.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[5][0] 6.0.5.1/24']],
                # route traffic to 2nd VM; Set DST MAC to MAC of 2nd (output) vhost user interfacei
                # of VM2, so traffic is accepted by VPP in next table
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[5][0] 6.0.5.2 00:00:00:00:00:12 fib-id 7']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 7 via 6.0.5.2']],
                # 3rd table
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[2][0] 8']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[6][0] 8']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[6][0] 6.0.6.1/24']],
                # route traffic via 2nd phy NIC to reach trafficgen
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.2.2 00:00:00:00:00:0A fib-id 8']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 8 via 6.0.2.2']],
                ['vswitch', 'run_vppctl', ['show ip fib']],
                ['vnf1', 'start'],
                ['vnf2', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
                ['vnf1', 'stop'],
                ['vnf2', 'stop'],
            ] +
            STEP_VSWITCH_PVVP_FINIT
    },
    {
        "Name": "p2p_l4_multi_PORT_arp_vpp",
        "Deployment": "clean",
        "Description": "VPP: P2P L4 multistream with unique ARP entry for each IP stream and port check",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'srcip': '6.0.2.2',
                    'dstip': '5.0.0.2',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 5.0.0.1/8']],
                ['tools', 'exec_python',
                 'import netaddr;'
                 'dst_mac_value = netaddr.EUI("00:00:00:00:00:0A").value;'
                 'cmds=open("/tmp/vppctl_cmds.txt","w");'
                 '[print("set ip arp #STEP[2][0] {} {}".format('
                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i),'
                 'netaddr.EUI(dst_mac_value+i,dialect=netaddr.mac_unix_expanded)),file=cmds) '
                 'for i in range($TRAFFIC["multistream"])];'
                 'cmds.close()'],
                ['vswitch', 'run_vppctl',
                 ['set ip source-and-port-range-check vrf 7 $TRAFFIC["l3"]["srcip"]/24 '
                  'port $TRAFFIC["l4"]["dstport"]']],
                ['vswitch', 'run_vppctl',
                 ['set interface ip source-and-port-range-check #STEP[1][0] udp-out-vrf 7']],
                ['tools', 'exec_shell', "rm -rf /tmp/vppctl_cmds_split*; split -l 1000 "
                                        "/tmp/vppctl_cmds.txt /tmp/vppctl_cmds_split"],
                ['tools', 'exec_shell', "for a in /tmp/vppctl_cmds_split* ; do echo $a ; "
                                        "sudo $TOOLS['vppctl'] exec $a ; sleep 2 ; done"],
                ['vswitch', 'run_vppctl', ['show ip fib summary']],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "p2p_l4_multi_PORT_mask_vpp",
        "Deployment": "clean",
        "Description": "VPP: P2P L4 multistream with 1 route for /8 net mask and port check",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'srcip': '6.0.2.2',
                    'dstip': '5.0.0.1',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']], # STEP 3
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.3.1/24']], # STEP 4
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.3.2 00:00:00:00:00:0A']], # STEP 5
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 via 6.0.3.2']],
                ['vswitch', 'run_vppctl', ['show ip fib']],
                ['vswitch', 'run_vppctl',
                 ['set ip source-and-port-range-check vrf 7 $TRAFFIC["l3"]["srcip"]/24 '
                  'port $TRAFFIC["l4"]["dstport"]']],
                ['vswitch', 'run_vppctl',
                 ['set interface ip source-and-port-range-check #STEP[1][0] udp-out-vrf 7']],
                ['vswitch', 'run_vppctl', ['show ip fib']],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "p2p_l4_multi_PORT_routes_vpp",
        "Deployment": "clean",
        "Description": "VPP: P2P L4 multistream with unique route for each IP stream and port check",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'srcip': '6.0.2.2',
                    'dstip': '5.0.0.1',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.3.1/24']],
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.3.2 00:00:00:00:00:0A']],
                # insertion of huge number of IP routes doesn't cause issues
                # seen with ARP entries at p2p_l3_multi_IP_vpp testcase
                ['vswitch', 'run_vppctl',
                 ['ip route add count $TRAFFIC["multistream"] $TRAFFIC["l3"]["dstip"]/32 via 6.0.3.2']],
                ['vswitch', 'run_vppctl',
                 ['set ip source-and-port-range-check vrf 7 $TRAFFIC["l3"]["srcip"]/24 '
                  'port $TRAFFIC["l4"]["dstport"]']],
                ['vswitch', 'run_vppctl',
                 ['set interface ip source-and-port-range-check #STEP[1][0] udp-out-vrf 7']],
                ['vswitch', 'run_vppctl', ['show ip fib summary']],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "pvp_l4_multi_PORT_mask_vpp",
        "Deployment": "clean",
        "Description": "VPP: PVP L4 multistream with route for /8 net and port mask",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "GUEST_TESTPMD_FWD_MODE" : ['io'],
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'srcip': '4.0.0.1',
                    'dstip': '5.0.0.1',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_PVP_INIT +
            [
                ['vswitch', 'run_vppctl', ['set int mac address #STEP[4][0] 00:00:00:00:00:11']],
                # create table for port check
                ['vswitch', 'run_vppctl', ['set ip source-and-port-range-check vrf 5 '
                                           '$TRAFFIC["l3"]["srcip"]/24 '
                  'port $TRAFFIC["l4"]["dstport"]']],
                # two separate tables are used, so the same IP network can be routed
                # via different DST IPs to reach VM and trafficgen via 2nd phy NIC
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[1][0] 6']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[3][0] 6']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.1.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[3][0] 6.0.3.1/24']],
                # enforce port check
                ['vswitch', 'run_vppctl', ['set int ip source-and-port-range-check #STEP[1][0] '
                                           'udp-out-vrf 5']],
                # route traffic to VM; Set DST MAC to MAC of 2nd (output) vhost user interface
                # of VM, so traffic is accepted by VPP in next table
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[3][0] 6.0.3.2 00:00:00:00:00:11 fib-id 6']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 6 via 6.0.3.2']],

                ['vswitch', 'run_vppctl', ['set int ip table #STEP[2][0] 7']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[4][0] 7']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[4][0] 6.0.4.1/24']],
                # route traffic via 2nd phy NIC to reach trafficgen
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.2.2 00:00:00:00:00:0A fib-id 7']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 7 via 6.0.2.2']],

                ['vswitch', 'run_vppctl', ['show ip fib']],
                ['vnf', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
                ['vnf', 'stop'],
            ] +
            STEP_VSWITCH_PVP_FINIT
    },
    {
        "Name": "pvvp_l4_multi_PORT_mask_vpp",
        "Deployment": "clean",
        "Description": "VPP: PVVP L4 multistream with route for /8 net and port mask",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "GUEST_TESTPMD_FWD_MODE" : ['io', 'io'],
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'srcip': '4.0.0.1',
                    'dstip': '5.0.0.1',
                },
                'l4': {
                    'enabled': True,
                    'srcport': 7,
                    'dstport': 8,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_PVVP_INIT +
            [
                ['vswitch', 'run_vppctl', ['set int mac address #STEP[4][0] 00:00:00:00:00:11']],
                ['vswitch', 'run_vppctl', ['set int mac address #STEP[6][0] 00:00:00:00:00:12']],
                # create table for port check
                ['vswitch', 'run_vppctl', ['set ip source-and-port-range-check vrf 5 '
                                           '$TRAFFIC["l3"]["srcip"]/24 port $TRAFFIC["l4"]["dstport"]']],
                # three separate tables are used, so the same IP network can be routed
                # via different DST IPs to reach both VMs and trafficgen via 2nd phy NIC
                # 1st table
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[1][0] 6']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[3][0] 6']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[1][0] 6.0.1.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[3][0] 6.0.3.1/24']],
                # enforce port check
                ['vswitch', 'run_vppctl', ['set int ip source-and-port-range-check #STEP[1][0] udp-out-vrf 5']],
                # route traffic to 1st VM; Set DST MAC to MAC of 2nd (output) vhost user interface
                # of VM1, so traffic is accepted by VPP in next table
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[3][0] 6.0.3.2 00:00:00:00:00:11 fib-id 6']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 6 via 6.0.3.2']],
                # 2nd table
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[4][0] 7']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[5][0] 7']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[4][0] 6.0.4.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[5][0] 6.0.5.1/24']],
                # route traffic to 2nd VM; Set DST MAC to MAC of 2nd (output) vhost user interfacei
                # of VM2, so traffic is accepted by VPP in next table
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[5][0] 6.0.5.2 00:00:00:00:00:12 fib-id 7']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 7 via 6.0.5.2']],
                # 3rd table
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[2][0] 8']],
                ['vswitch', 'run_vppctl', ['set int ip table #STEP[6][0] 8']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[2][0] 6.0.2.1/24']],
                ['vswitch', 'run_vppctl', ['set int ip address #STEP[6][0] 6.0.6.1/24']],
                # route traffic via 2nd phy NIC to reach trafficgen
                ['vswitch', 'run_vppctl', ['set ip arp #STEP[2][0] 6.0.2.2 00:00:00:00:00:0A fib-id 8']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 table 8 via 6.0.2.2']],
                ['vswitch', 'run_vppctl', ['show ip fib']],
                ['vnf1', 'start'],
                ['vnf2', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
                ['vnf1', 'stop'],
                ['vnf2', 'stop'],
            ] +
            STEP_VSWITCH_PVVP_FINIT
    },
    #
    # End of L3 & L4 tests to compare OVS and VPP performance
    #
    #
    # VxLAN tests to compare OVS and VPP performance
    #
    # Example of execution:
    #   ./vsperf --test-params "TRAFFIC={'multistream':2000,'traffic_type':'rfc2544_continuous'}" \
    #       vxlan_multi_IP_vpp
    {
        "Name": "vxlan_multi_IP_mask_ovs",
        "Deployment": "op2p",
        "Tunnel Type": "vxlan",
        "Tunnel Operation": "encapsulation",
        "Description": "OVS: VxLAN L3 multistream",
        "Parameters": {
            "TRAFFICGEN_IXNET_TCL_SCRIPT" : "ixnetrfc2544v2.tcl",
            "TRAFFICGEN_PORT1_MAC" : '00:00:00:00:00:01',
            "TRAFFICGEN_PORT2_MAC" : '#PARAM(NICS[0]["mac"])',
            "TRAFFICGEN_PORT1_IP"  : '9.0.0.2',
            "TRAFFICGEN_PORT2_IP"  : '10.0.0.2',
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
            },
        },
        "TestSteps": [
            ['vswitch', 'del_flow', 'br0', {'in_port': '1'}],
            ['vswitch', 'add_flow', 'br0',
             {'in_port': '1', 'dl_type': '0x800', 'nw_proto': '17',
              'nw_dst': '$TRAFFICGEN_PORT2_IP/8', 'actions': ['output:2'],
              'idle_timeout': '0'}],
            ['vswitch', 'dump_flows', 'br0'],
            ['vswitch', 'dump_flows', 'br-ext'],
        ],
    },
    {
        "Name": "vxlan_multi_IP_arp_vpp",
        "Deployment": "clean",
        "Description": "VPP: VxLAN L3 multistream with unique ARP entry for each IP stream",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '10.0.0.2',
                },
                'l4': {
                    'enabled': True,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'run_vppctl',
                 ['set int ip address #STEP[1][0] 9.0.0.1/16']],
                ['vswitch', 'run_vppctl',
                 ['set int ip address #STEP[2][0] 11.0.0.1/16']],
                ['vswitch', 'run_vppctl',
                 ['create vxlan tunnel src 11.0.0.1 dst 11.0.0.2 vni 1']],
                ['vswitch', 'run_vppctl',
                 ['set int state vxlan_tunnel0 up']],
                ['vswitch', 'run_vppctl',
                 ['set int ip address vxlan_tunnel0 10.0.0.1/8']],
                ['vswitch', 'run_vppctl',
                 ['set ip arp #STEP[2][0] 11.0.0.2 00:00:00:00:00:00']],
                ['tools', 'exec_python',
                 'import netaddr;'
                 'dst_mac_value = netaddr.EUI("00:00:00:00:00:0A").value;'
                 'cmds=open("/tmp/vppctl_cmds.txt","w");'
                 '[print("set ip arp vxlan_tunnel0 {} {}".format('
                 'netaddr.IPAddress(netaddr.IPAddress("$TRAFFIC["l3"]["dstip"]").value+i),'
                 'netaddr.EUI(dst_mac_value+i,dialect=netaddr.mac_unix_expanded)),file=cmds) '
                 'for i in range($TRAFFIC["multistream"])];'
                 'cmds.close()'],
                ['tools', 'exec_shell',
                 "rm -rf /tmp/vppctl_cmds_split*;"
                 "split -l 1000 /tmp/vppctl_cmds.txt /tmp/vppctl_cmds_split"],
                ['tools', 'exec_shell',
                 "for a in /tmp/vppctl_cmds_split* ; do "
                 "echo $a ; sudo $TOOLS['vppctl'] exec $a ; sleep 2 ; done"],
                ['vswitch', 'run_vppctl', ['show ip fib summary']],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    {
        "Name": "vxlan_multi_IP_mask_vpp",
        "Deployment": "clean",
        "Description": "VPP: VxLAN L3 multistream with 1 route for /8 netmask",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                'bidir' : 'False',
                "stream_type" : "L3",
                'l2': {
                    'dstmac' : '#PARAM(NICS[0]["mac"])',
                },
                'l3': {
                    'enabled': True,
                    'proto': 'udp',
                    'dstip': '8.0.0.1',
                },
                'l4': {
                    'enabled': True,
                },
            },
        },
        "TestSteps":
            STEP_VSWITCH_P2P_INIT +
            [
                ['vswitch', 'run_vppctl',
                 ['set int ip address #STEP[1][0] 9.0.0.1/16']],
                ['vswitch', 'run_vppctl',
                 ['set int ip address #STEP[2][0] 11.0.0.1/16']],
                ['vswitch', 'run_vppctl',
                 ['create vxlan tunnel src 11.0.0.1 dst 11.0.0.2 vni 1']],
                ['vswitch', 'run_vppctl',
                 ['set int state vxlan_tunnel0 up']],
                ['vswitch', 'run_vppctl',
                 ['set int ip address vxlan_tunnel0 10.0.0.1/24']],
                ['vswitch', 'run_vppctl',
                 ['set ip arp #STEP[2][0] 11.0.0.2 00:00:00:00:00:00']],
                ['vswitch', 'run_vppctl',
                 ['set ip arp vxlan_tunnel0 10.0.0.2 00:00:00:00:00:0A']],
                ['vswitch', 'run_vppctl', ['ip route add $TRAFFIC["l3"]["dstip"]/8 via 10.0.0.2']],
                ['vswitch', 'run_vppctl', ['show ip fib']],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'run_vppctl', ['show interfaces']],
            ] +
            STEP_VSWITCH_P2P_FINIT
    },
    #
    # End of VxLAN tests to compare OVS and VPP performance
    #
]