conf/01_testcases.conf


1
2
3
4
5
6
7
8
9
10
11
1Project: L2/L3 forwarding performance toolkit for NFVi (NFVbench)
Project Creation Date: May 2nd 2017
Project Category:
Lifecycle State:
Primary Contact: Alec Hothan (ahothan@cisco.com)
Project Lead: Alec Hothan (ahothan@cisco.com)
Jira Project Name: L2 L3 forwarding performance toolkit for NFVi
Jira Project Prefix: NFVBENCH
Mailing list tag: [nfvbench]
IRC: Server:
Repository: nfvbench

Committers:
Alec Hothan (ahothan@cisco.com)
Carsten Rossenhövel (cross@eantc.com)
Frank Brockners (fbrockne@cisco.com)
Yichen Wang (yicwang@cisco.com)
Al Morton (acmorton@att.com)

Link to TSC approval of the project:

Acknowledgements
The development of NFVbench started in Summer 2016 at Cisco by this small team of dedicated people
before being open sourced in Spring 2017 to OPNFV following more than 500 commits:
 Jan Balaz (aka Johnny)
 Stefano Chiesa Suryanto
 Yichen Wang
 Alec Hothan
# Copyright 2015-2017 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file describes a list of testcases.  Each testcase is described as a
# dictionary in a list of dictionaries.
#
# The dictionary keys, their meanings and available values are:
#
# "Name": "phy2phy_burst",         # A human-readable string identifying the
#                                  # test.
# "Deployment": "p2p",             # One of the supported deployment scenarios.
# "Description": "Lorem ipsum..."  # Optional. A human-readable string
#                                  # describing the test.
# "Frame Modification": "vlan"     # One of the supported frame modifications:
#                                  # vlan, mpls, mac, dscp, ttl, ip_addr,
#                                  # ip_port.
# "Load": dictionary               # Optional. Configures background load
#                                  # during testcase execution.
#                                  # The tool used to generate load is
#                                  # defined  by LOADGEN configuration.
#                                  # Default setting of Dummy can be found in
#                                  # 07_loadgen.conf.
#   Description of "Load" dictionary keys, their meanings and available values:
#
#   "load": 0-100                  # percentage of cores which should be
#                                  # utilized by load generator
#                                  # e.g. load = 70%, detected cpu cores = 14 =>
#                                  # round(14*0.7)=10, i.e. 10 instances of load
#                                  # generator will be executed
#   "reserved": 0- (Default 0)     # Optional. Defines number of cores reserved
#                                  # for vsperf
#                                  # e.g. load = 80%, detected cpu cores = 14,
#                                  # reserved = 4 => round((14-4)*0.8)=8,
#                                  # i.e. 8 load gen. instances will be executed
#   "pattern" : "c"                # stress/stress-ng specific; Number of 'c',
#                                  # 'm' and 'i' defines ratio between cpu, mem
#                                  # and io workers respectively
#                                  # e.g. "ccccmmi" => ratio among workers types
#                                  # will be 3:2:1, so in case that 12 stress
#                                  # instances should be executed, then 6 cpu,
#                                  # 4 memory and 2 io workers will be executed
#   "load_memory": 0-100           # Optional. Defines percentage of the system
#                                  # memory, which should be utilized by memory
#                                  # workers (if they are part of "pattern").
#                                  # if not specified then default stress(-ng)
#                                  # value will be used
#   "options": ""                  # Optional. Additional command line options
#                                  # to be passed to the load generator.
# "vSwitch" : "OvsVanilla"         # Defines vSwitch to be used for test execution.
#                                  # It will override any VSWITCH option stated
#                                  # in configuration files or value specified
#                                  # on command line through --vswitch parameter.
# "VNF" : "QemuVirtioNet"          # Defines VNF to be used for test execution.
#                                  # It will override any VNF option stated
#                                  # in configuration files or value specified
#                                  # on command line through --vnf parameter.
# "Trafficgen" : "Dummy"           # Defines traffic generator to be used for test
#                                  # execution. It will override any VNF option
#                                  # stated in configuration files or value
#                                  # specified on command line through --trafficgen
#                                  # parameter.
# "Parameters" : {'TRAFFICGEN_PKT_SIZES' : (512,)},
#                                  # Dictionary with testcase specific configuration
#                                  # environment. Specified parameters will be modified
#                                  # before the test execution and their original values will
#                                  # be restored after TC finishes. This dictionary will
#                                  # override any values defined by TEST_PARAMS option
#                                  # stated in configuration files or values specified
#                                  # on command line through --test-params parameter.
#
# "TestSteps": []                  # Definition of detailed test steps.
#                                  # In case that this list is defined, then
#                                  # vsperf will execute defined test steps
#                                  # one by one. It can be used to configure
#                                  # vswitch, insert flows and transmit traffic.
#                                  # It is possible to refer to result of any
#                                  # previous step through #STEP[i][j] macro.
#                                  # Where i is a number of step (starts from 0)
#                                  # and j is index of result returned by step i.
# "Test Modifier": [FrameMod|Other],
# "Dependency": [Test_Case_Name |None],

#
# VPP specific macros used in TC defintions
#
VPP_P2P =   [
                ['vswitch', 'add_switch', 'int_br0'],           # STEP 0
                ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 1
                ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 2
                ['vswitch', 'add_connection', 'int_br0', '#STEP[1][0]', '#STEP[2][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[2][0]', '#STEP[1][0]', True],
                ['trafficgen', 'send_traffic', {}],
                ['vswitch', 'dump_connections', 'int_br0'],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[1][0]', '#STEP[2][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[2][0]', '#STEP[1][0]', True],
                ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
                ['vswitch', 'del_switch', 'int_br0'],
            ]
VPP_PVP =   [
                ['vswitch', 'add_switch', 'int_br0'],           # STEP 0
                ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 1
                ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 2
                ['vswitch', 'add_vport', 'int_br0'],            # STEP 3
                ['vswitch', 'add_vport', 'int_br0'],            # STEP 4
                ['vswitch', 'add_connection', 'int_br0', '#STEP[1][0]', '#STEP[3][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[4][0]', '#STEP[2][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[2][0]', '#STEP[4][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[3][0]', '#STEP[1][0]', True],
                ['vnf', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vnf', 'stop'],
                ['vswitch', 'dump_connections', 'int_br0'],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[1][0]', '#STEP[3][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[4][0]', '#STEP[2][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[2][0]', '#STEP[4][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[3][0]', '#STEP[1][0]', True],
                ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'],
                ['vswitch', 'del_switch', 'int_br0'],
            ]
VPP_PVVP =   [
                ['vswitch', 'add_switch', 'int_br0'],           # STEP 0
                ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 1
                ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 2
                ['vswitch', 'add_vport', 'int_br0'],            # STEP 3
                ['vswitch', 'add_vport', 'int_br0'],            # STEP 4
                ['vswitch', 'add_vport', 'int_br0'],            # STEP 5
                ['vswitch', 'add_vport', 'int_br0'],            # STEP 6
                ['vswitch', 'add_connection', 'int_br0', '#STEP[1][0]', '#STEP[3][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[4][0]', '#STEP[5][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[6][0]', '#STEP[2][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[2][0]', '#STEP[6][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[5][0]', '#STEP[4][0]', True],
                ['vswitch', 'add_connection', 'int_br0', '#STEP[3][0]', '#STEP[1][0]', True],
                ['vnf1', 'start'],
                ['vnf2', 'start'],
                ['trafficgen', 'send_traffic', {}],
                ['vnf2', 'stop'],
                ['vnf1', 'stop'],
                ['vswitch', 'dump_connections', 'int_br0'],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[1][0]', '#STEP[3][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[4][0]', '#STEP[5][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[6][0]', '#STEP[2][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[2][0]', '#STEP[6][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[5][0]', '#STEP[4][0]', True],
                ['vswitch', 'del_connection', 'int_br0', '#STEP[3][0]', '#STEP[1][0]', True],
                ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[5][0]'],
                ['vswitch', 'del_port', 'int_br0', '#STEP[6][0]'],
                ['vswitch', 'del_switch', 'int_br0'],
            ]

#
# Generic performance TC definitions
#
PERFORMANCE_TESTS = [
    {
        "Name": "phy2phy_tput",
        "Deployment": "p2p",
        "Description": "LTD.Throughput.RFC2544.PacketLossRatio",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
    },
    {
        "Name": "phy2phy_forwarding",
        "Deployment": "p2p",
        "Description": "LTD.Forwarding.RFC2889.MaxForwardingRate",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2889_forwarding",
            },
        },
    },
    {
        "Name": "phy2phy_learning",
        "Deployment": "p2p",
        "Description": "LTD.AddrLearning.RFC2889.AddrLearningRate",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2889_learning",
            },
        },
    },
    {
        "Name": "phy2phy_caching",
        "Deployment": "p2p",
        "Description": "LTD.AddrCaching.RFC2889.AddrCachingCapacity",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2889_caching",
            },
        },
    },
    {
        "Name": "back2back",
        "Deployment": "p2p",
        "Description": "LTD.Throughput.RFC2544.BackToBackFrames",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_back2back",
            },
        },
    },
    {
        "Name": "phy2phy_tput_mod_vlan",
        "Deployment": "p2p",
        "Frame Modification": "vlan",
        "Description": "LTD.Throughput.RFC2544.PacketLossRatioFrameModification",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
    },
    {
        "Name": "phy2phy_cont",
        "Deployment": "p2p",
        "Description": "Phy2Phy Continuous Stream",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_continuous",
                "frame_rate" : 100,
            },
        },
    },
    {
        "Name": "pvp_cont",
        "Deployment": "pvp",
        "Description": "PVP Continuous Stream",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_continuous",
                "frame_rate" : 100,
            },
        },
    },
    {
        "Name": "pvvp_cont",
        "Deployment": "pvvp",
        "Description": "PVVP Continuous Stream",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_continuous",
                "frame_rate" : 100,
            },
        },
    },
    {
        "Name": "pvpv_cont",
        "Deployment": "pvpv",
        "Description": "Two VMs in parallel with Continuous Stream",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_continuous",
                "frame_rate" : 100,
            },
        },
    },
    {
        "Name": "phy2phy_scalability",
        "Deployment": "p2p",
        "Description": "LTD.Scalability.Flows.RFC2544.0PacketLoss",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
                "multistream" : 8000,
            },
        },
    },
    {
        "Name": "pvp_tput",
        "Deployment": "pvp",
        "Description": "LTD.Throughput.RFC2544.PacketLossRatio",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
    },
    {
        "Name": "pvp_back2back",
        "Deployment": "pvp",
        "Description": "LTD.Throughput.RFC2544.BackToBackFrames",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_back2back",
            },
        },
    },
    {
        "Name": "pvvp_tput",
        "Collector": "cpu",
        "Deployment": "pvvp",
        "Description": "LTD.Throughput.RFC2544.PacketLossRatio",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
    },
    {
        "Name": "pvvp_back2back",
        "Collector": "cpu",
        "Deployment": "pvvp",
        "Description": "LTD.Throughput.RFC2544.BackToBackFrames",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_back2back",
            },
        },
    },
    {
        "Name": "phy2phy_cpu_load",
        "Deployment": "p2p",
        "Description": "LTD.CPU.RFC2544.0PacketLoss",
        "Load" : {
            "load" : 100,
            "reserved" : 4,
            "pattern" : "c",
        },
        "Parameters" : {
            "LOADGEN" : "StressNg",
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
    },
    {
        "Name": "phy2phy_mem_load",
        "Deployment": "p2p",
        "Description": "LTD.Memory.RFC2544.0PacketLoss",
        "Load" : {
            "load" : 50,
            "pattern" : "m",
            "load_memory" : 80,
        },
        "Parameters" : {
            "LOADGEN" : "StressNg",
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
    },
    {
        "Name": "phy2phy_tput_vpp",
        "Deployment": "clean",
        "Description": "VPP: LTD.Throughput.RFC2544.PacketLossRatio",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
        "TestSteps": VPP_P2P,
    },
    {
        "Name": "phy2phy_cont_vpp",
        "Deployment": "clean",
        "Description": "VPP: Phy2Phy Continuous Stream",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_continuous",
                "frame_rate" : 100,
            },
        },
        "TestSteps": VPP_P2P,
    },
    {
        "Name": "phy2phy_back2back_vpp",
        "Deployment": "clean",
        "Description": "VPP: LTD.Throughput.RFC2544.BackToBackFrames",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_back2back",
            },
        },
        "TestSteps": VPP_P2P,
    },
    {
        "Name": "pvp_tput_vpp",
        "Deployment": "clean",
        "Description": "VPP: LTD.Throughput.RFC2544.PacketLossRatio",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
        "TestSteps": VPP_PVP,
    },
    {
        "Name": "pvp_cont_vpp",
        "Deployment": "clean",
        "Description": "VPP: PVP Continuous Stream",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_continuous",
            },
        },
        "TestSteps": VPP_PVP,
    },
    {
        "Name": "pvp_back2back_vpp",
        "Deployment": "clean",
        "Description": "VPP: LTD.Throughput.RFC2544.BackToBackFrames",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_back2back",
            },
        },
        "TestSteps": VPP_PVP,
    },
    {
        "Name": "pvvp_tput_vpp",
        "Deployment": "clean",
        "Description": "VPP: LTD.Throughput.RFC2544.PacketLossRatio",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_throughput",
            },
        },
        "TestSteps": VPP_PVVP,
    },
    {
        "Name": "pvvp_cont_vpp",
        "Deployment": "clean",
        "Description": "VPP: PVP Continuous Stream",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_continuous",
            },
        },
        "TestSteps": VPP_PVVP,
    },
    {
        "Name": "pvvp_back2back_vpp",
        "Deployment": "clean",
        "Description": "VPP: LTD.Throughput.RFC2544.BackToBackFrames",
        "vSwitch" : "VppDpdkVhost",
        "Parameters" : {
            "TRAFFIC" : {
                "traffic_type" : "rfc2544_back2back",
            },
        },
        "TestSteps": VPP_PVVP,
    },
]