From eabc66eef336b3c47c366027b205d26db10a3c21 Mon Sep 17 00:00:00 2001 From: Martin Klozik Date: Mon, 24 Oct 2016 12:37:26 +0100 Subject: cli: Modify configuration via CLI In the past, only a few configuration parameters could be modified via --test-params CLI argument and it was not easy to find out their complete list. This patch adds support for generic modification of any configuration parameter via CLI argument --test-params or by "Parameters" section of testcase definition. Thus it is possible to customize a vsperf configuration environment per testcase or for each vsperf execution. Old CLI parameters duration, pkt_sizes, rfc2544_tests and rfc2889_trials were renamed to TRAFFICGEN_DURATION, TRAFFICGEN_PKT_SIZES, TRAFFICGEN_RFC2544_TESTS and TRAFFICGEN_RFC2889_TRIALS to be consistent with other configuration parameters. JIRA: VSPERF-375 Change-Id: I50a1f4ff7250d754aa8af0295a9c7c1be8151175 Signed-off-by: Martin Klozik Reviewed-by: Al Morton Reviewed-by: Christian Trautman Reviewed-by: Bill Michalowski Reviewed-by: Antonio Fischetti Reviewed-by: --- ci/build-vsperf.sh | 2 +- conf/01_testcases.conf | 13 +++-- conf/03_traffic.conf | 19 ++++++++ conf/04_vnf.conf | 2 - conf/10_custom.conf | 2 +- conf/__init__.py | 78 +++++++++++++++++++++++++----- conf/integration/01_testcases.conf | 19 ++++++-- conf/integration/03_traffic.conf | 3 -- core/traffic_controller.py | 17 ++----- core/traffic_controller_rfc2544.py | 5 +- core/traffic_controller_rfc2889.py | 7 ++- docs/configguide/trafficgen.rst | 5 +- docs/design/vswitchperf_design.rst | 3 +- docs/requirements/vswitchperf_ltp.rst | 2 +- docs/userguide/integration.rst | 20 ++++---- docs/userguide/teststeps.rst | 4 +- docs/userguide/testusage.rst | 90 +++++++++++++++++++++++++---------- testcases/testcase.py | 23 ++++++--- tools/pkt_gen/ixnet/ixnet.py | 18 ++++--- vnfs/qemu/qemu.py | 34 ++++--------- vnfs/qemu/qemu_dpdk_vhost_user.py | 5 -- vsperf | 45 +++++++----------- vswitches/ovs_dpdk_vhost.py | 11 +---- 23 files changed, 257 insertions(+), 170 deletions(-) diff --git a/ci/build-vsperf.sh b/ci/build-vsperf.sh index 637bc474..38fda03b 100755 --- a/ci/build-vsperf.sh +++ b/ci/build-vsperf.sh @@ -49,7 +49,7 @@ TESTCASES_MERGE="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_ TESTPARAM_MERGE="--integration" # DAILY - run selected TCs for defined packet sizes TESTCASES_DAILY='phy2phy_tput back2back phy2phy_tput_mod_vlan phy2phy_scalability pvp_tput pvp_back2back pvvp_tput pvvp_back2back' -TESTPARAM_DAILY='--test-params pkt_sizes=64,128,512,1024,1518' +TESTPARAM_DAILY='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)' # check if user config file exists if not then we will use default settings if [ -f $HOME/vsperf-${BRANCH}.conf ] ; then # branch specific config was found diff --git a/conf/01_testcases.conf b/conf/01_testcases.conf index 55cce1cf..f30c1912 100755 --- a/conf/01_testcases.conf +++ b/conf/01_testcases.conf @@ -115,11 +115,14 @@ # # stated in configuration files or value # # specified on command line through --trafficgen # # parameter. -# "Parameters" : "pkt_sizes=512" # Defines list of test parameters used for test -# # execution. It will override any values defined -# # by TEST_PARAMS option stated in configuration -# # files or values specified on command line through -# # --test-params parameter. +# "Parameters" : {'TRAFFICGEN_PKT_SIZES' : (512,)} +# # Dictionary with testcase specific configuration +# # environment. Specified parameters will be modified +# # before the test execution and their original values will +# # be restored after TC finishes. This dictionary will +# # override any values defined by TEST_PARAMS option +# # stated in configuration files or values specified +# # on command line through --test-params parameter. # "Test Modifier": [FrameMod|Other], # "Dependency": [Test_Case_Name |None], diff --git a/conf/03_traffic.conf b/conf/03_traffic.conf index 8efd5aab..d7327629 100644 --- a/conf/03_traffic.conf +++ b/conf/03_traffic.conf @@ -34,6 +34,15 @@ TRAFFICGEN = 'Dummy' # Expand like this: (64, 128, 256, 512, 1024) TRAFFICGEN_PKT_SIZES = (64,) +TRAFFICGEN_DURATION = 30 + +TRAFFICGEN_RFC2544_TESTS = 1 +TRAFFICGEN_RFC2889_TRIALS = 1 +TRAFFICGEN_LOSSRATE = 0.0 + +############################# +# IXIA Configuration -- BEGIN + # path to 'ixos' install path TRAFFICGEN_IXIA_ROOT_DIR = '/opt/ixos' @@ -67,6 +76,10 @@ TRAFFICGEN_IXIA_3RD_PARTY = os.path.join(ROOT_DIR, '3rd_party/ixia') # default TCL script, which will be used for IXNETWORK configuration TRAFFICGEN_IXNET_TCL_SCRIPT = 'ixnetrfc2544.tcl' +# IXIA Configuration -- END +########################### + + ########################################### # Spirent TestCenter Configuration -- BEGIN @@ -172,6 +185,9 @@ TRAFFICGEN_STC_VERBOSE = "True" # Spirent TestCenter Configuration -- END ######################################### +############################# +# Xena Configuration -- BEGIN + # Xena traffic generator connection info TRAFFICGEN_XENA_IP = '' TRAFFICGEN_XENA_PORT1 = '' @@ -198,6 +214,9 @@ TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION = '0.5' TRAFFICGEN_XENA_2544_TPUT_USEPASS_THRESHHOLD = 'false' TRAFFICGEN_XENA_2544_TPUT_PASS_THRESHHOLD = '0.0' +# Xena Configuration -- END +########################### + ################################################### # MoonGen Configuration and Connection Info-- BEGIN diff --git a/conf/04_vnf.conf b/conf/04_vnf.conf index d3b4be4d..75f107e8 100644 --- a/conf/04_vnf.conf +++ b/conf/04_vnf.conf @@ -96,8 +96,6 @@ GUEST_SHARED_DRIVE_TYPE = ['scsi'] # 'linux_bridge' - linux bridge will be configured # 'buildin' - nothing will be configured by vsperf; VM image must # ensure traffic forwarding between its interfaces -# This configuration option can be overridden by CLI SCALAR option -# guest_loopback, e.g. --test-params "guest_loopback=l2fwd" # For 2 VNFs you may use ['testpmd', 'l2fwd'] GUEST_LOOPBACK = ['testpmd'] diff --git a/conf/10_custom.conf b/conf/10_custom.conf index b3707149..38875810 100644 --- a/conf/10_custom.conf +++ b/conf/10_custom.conf @@ -103,7 +103,7 @@ TRAFFICGEN_MOONGEN_LINE_SPEED_GBPS = '10' # MoonGen Configuration and Connection Info-- END ################################################### -#TEST_PARAMS = {'pkt_sizes':'64'} +#TEST_PARAMS = {'TRAFFICGEN_PKT_SIZES':(64,)} OPNFV_INSTALLER = "Fuel" OPNFV_URL = "http://testresults.opnfv.org/testapi" PACKAGE_LIST = "src/package-list.mk" diff --git a/conf/__init__.py b/conf/__init__.py index 4d6f57fe..2448d390 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -29,6 +29,10 @@ import netaddr _LOGGER = logging.getLogger(__name__) +# Special test parameters which are not part of standard VSPERF configuration +_EXTRA_TEST_PARAMS = ['bidirectional', 'traffic_type', 'iload', 'tunnel_type', + 'multistream', 'stream_type', 'pre-installed_flows'] + # regex to parse configuration macros from 04_vnf.conf # it will select all patterns starting with # sign # and returns macro parameters and step @@ -49,7 +53,13 @@ class Settings(object): """Return a settings item value """ if attr in self.__dict__: - return getattr(self, attr) + if attr == 'TEST_PARAMS': + return getattr(self, attr) + else: + master_value = getattr(self, attr) + # Check if parameter value was overridden by CLI option + cli_value = get_test_param(attr, None) + return cli_value if cli_value else master_value else: raise AttributeError("%r object has no attribute %r" % (self.__class__, attr)) @@ -136,6 +146,24 @@ class Settings(object): for key in os.environ: setattr(self, key, os.environ[key]) + def check_test_params(self): + """ + Check all parameters defined inside TEST_PARAMS for their + existence. In case that non existing vsperf parmeter name + is detected, then VSPER will raise a runtime error. + """ + unknown_keys = [] + for key in settings.getValue('TEST_PARAMS'): + if key == 'TEST_PARAMS': + raise RuntimeError('It is not allowed to define TEST_PARAMS ' + 'as a test parameter') + if key not in self.__dict__ and key not in _EXTRA_TEST_PARAMS: + unknown_keys.append(key) + + if len(unknown_keys): + raise RuntimeError('Test parameters contain unknown configuration ' + 'parameter(s): {}'.format(', '.join(unknown_keys))) + def check_vm_settings(self, vm_number): """ Check all VM related settings starting with GUEST_ prefix. @@ -145,15 +173,12 @@ class Settings(object): """ for key in self.__dict__: if key.startswith('GUEST_'): - if (isinstance(self.__dict__[key], str) and - self.__dict__[key].find('#') >= 0): - self.__dict__[key] = [self.__dict__[key]] + value = self.getValue(key) + if isinstance(value, str) and value.find('#') >= 0: self._expand_vm_settings(key, 1) - self.__dict__[key] = self.__dict__[key][0] - if isinstance(self.__dict__[key], list): - if (len(self.__dict__[key]) < vm_number or - str(self.__dict__[key][0]).find('#') >= 0): + if isinstance(value, list): + if len(value) < vm_number or str(value[0]).find('#') >= 0: # expand configuration for all VMs self._expand_vm_settings(key, vm_number) @@ -161,7 +186,15 @@ class Settings(object): """ Expand VM option with given key for given number of VMs """ - master_value = self.__dict__[key][0] + tmp_value = self.getValue(key) + if isinstance(tmp_value, str): + scalar = True + master_value = tmp_value + tmp_value = [tmp_value] + else: + scalar = False + master_value = tmp_value[0] + master_value_str = str(master_value) if master_value_str.find('#') >= 0: self.__dict__[key] = [] @@ -190,9 +223,12 @@ class Settings(object): value = ast.literal_eval(value) self.__dict__[key].append(value) else: - for vmindex in range(len(self.__dict__[key]), vm_number): + for vmindex in range(len(tmp_value), vm_number): self.__dict__[key].append(master_value) + if scalar: + self.__dict__[key] = self.__dict__[key][0] + _LOGGER.debug("Expanding option: %s = %s", key, self.__dict__[key]) def __str__(self): @@ -203,7 +239,11 @@ class Settings(object): Returns: A human-readable string. """ - return pprint.pformat(self.__dict__) + tmp_dict = {} + for key in self.__dict__: + tmp_dict[key] = self.getValue(key) + + return pprint.pformat(tmp_dict) # # validation methods used by step driven testcases @@ -222,7 +262,6 @@ class Settings(object): settings = Settings() - def get_test_param(key, default=None): """Retrieve value for test param ``key`` if available. @@ -232,4 +271,17 @@ def get_test_param(key, default=None): :returns: Value for ``key`` if found, else ``default``. """ test_params = settings.getValue('TEST_PARAMS') - return test_params.get(key, default) if test_params else default + if key in test_params: + if not isinstance(test_params.get(key), str): + return test_params.get(key) + else: + # values are passed inside string from CLI, so we must retype them accordingly + try: + return ast.literal_eval(test_params.get(key)) + except ValueError: + # for backward compatibility, we have to accept strings without quotes + _LOGGER.warning("Adding missing quotes around string value: %s = %s", + key, str(test_params.get(key))) + return str(test_params.get(key)) + else: + return default diff --git a/conf/integration/01_testcases.conf b/conf/integration/01_testcases.conf index 83d420bf..a584845a 100644 --- a/conf/integration/01_testcases.conf +++ b/conf/integration/01_testcases.conf @@ -328,6 +328,7 @@ INTEGRATION_TESTS = [ "The encap and decap are performed inside the " "virtual switch itself in each direction to avoid " "the need of ingress overlay traffic."), + "Parameters": {'TRAFFICGEN_IXNET_TCL_SCRIPT' : 'ixnetrfc2544v2.tcl'}, }, { "Name": "overlay_p2p_tput", @@ -337,6 +338,7 @@ INTEGRATION_TESTS = [ "Tunnel Type": SUPPORTED_TUNNELING_PROTO[0], "Tunnel Operation": "encapsulation", "Description": "Overlay Encapsulation Throughput RFC2544 Test", + "Parameters": {'TRAFFICGEN_IXNET_TCL_SCRIPT' : 'ixnetrfc2544v2.tcl'}, }, { "Name": "overlay_p2p_cont", @@ -346,6 +348,7 @@ INTEGRATION_TESTS = [ "Tunnel Type": SUPPORTED_TUNNELING_PROTO[0], "Tunnel Operation": "encapsulation", "Description": "Overlay Encapsulation Continuous Stream", + "Parameters": {'TRAFFICGEN_IXNET_TCL_SCRIPT' : 'ixnetrfc2544v2.tcl'}, }, { "Name": "overlay_p2p_decap_tput", @@ -355,6 +358,7 @@ INTEGRATION_TESTS = [ "Tunnel Type": SUPPORTED_TUNNELING_PROTO[0], "Tunnel Operation": "decapsulation", "Description": "Overlay Decapsulation Throughput RFC2544 Test", + "Parameters": {'TRAFFICGEN_IXNET_TCL_SCRIPT' : 'ixnetrfc2544v2.tcl'}, }, { "Name": "overlay_p2p_decap_cont", @@ -364,6 +368,7 @@ INTEGRATION_TESTS = [ "Tunnel Type": SUPPORTED_TUNNELING_PROTO[0], "Tunnel Operation": "decapsulation", "Description": "Overlay Decapsulation Continuous Stream", + "Parameters": {'TRAFFICGEN_IXNET_TCL_SCRIPT' : 'ixnetrfc2544v2.tcl'}, }, { "Name": "vswitch_add_del_bridge", @@ -747,7 +752,9 @@ INTEGRATION_TESTS = [ { # Topology: 2 Parallel PVP connections # To run a Linux bridge as a loopback in the Guest use: - # --test-params "guest_loopback=linux_bridge" --integration 2pvp_udp_dest_flows + # --test-params "GUEST_LOOPBACK=['linux_bridge']" --integration 2pvp_udp_dest_flows + # or add "Parameters" option to the test definition: + # "Parameters" : {'GUEST_LOOPBACK' : ['linux_bridge'],}, "Name": "2pvp_udp_dest_flows", "Description": "Continuous TC with 2 Parallel VMs, flows on UDP Dest Port", "Deployment": "clean", @@ -768,7 +775,9 @@ INTEGRATION_TESTS = [ { # Topology: 4 Parallel PVP connections # To run a Linux bridge as a loopback in the Guest use: - # --test-params "guest_loopback=linux_bridge" --integration 4pvp_udp_dest_flows + # --test-params "GUEST_LOOPBACK=['linux_bridge']" --integration 4pvp_udp_dest_flows + # or add "Parameters" option to the test definition: + # "Parameters" : {'GUEST_LOOPBACK' : ['linux_bridge'],}, "Name": "4pvp_udp_dest_flows", "Description": "Continuous TC with 4 Parallel VMs, flows on UDP Dest Port", "Deployment": "clean", @@ -793,7 +802,9 @@ INTEGRATION_TESTS = [ { # Topology: 6 Parallel PVP connections # To run a Linux bridge as a loopback in the Guest use: - # --test-params "guest_loopback=linux_bridge" --integration 6pvp_udp_dest_flows + # --test-params "GUEST_LOOPBACK=['linux_bridge']" --integration 6pvp_udp_dest_flows + # or add "Parameters" option to the test definition: + # "Parameters" : {'GUEST_LOOPBACK' : ['linux_bridge'],}, "Name": "6pvp_udp_dest_flows", "Description": "Continuous TC with 6 Parallel VMs, flows on UDP Dest Port", "Deployment": "clean", @@ -901,7 +912,7 @@ INTEGRATION_TESTS = [ # "vSwitch" : "OvsVanilla", # "VNF" : "QemuVirtioNet", # "Trafficgen": "IxNet", -# "Parameters": {"guest_loopback" : "linux_bridge"}, +# "Parameters": {"GUEST_LOOPBACK" : ["linux_bridge"],}, # "TestSteps": STEP_VSWITCH_PVP_FLOWS_INIT + # [ # ['vnf', 'start'], diff --git a/conf/integration/03_traffic.conf b/conf/integration/03_traffic.conf index 0b46cea0..e78e2668 100644 --- a/conf/integration/03_traffic.conf +++ b/conf/integration/03_traffic.conf @@ -18,9 +18,6 @@ TRAFFICGEN_PORT2_MAC = '02:00:00:00:00:02' TRAFFICGEN_PORT1_IP = '1.1.1.1' TRAFFICGEN_PORT2_IP = '90.90.90.90' -# To test VXLAN set the ff to ixnetrfc2544v2.tcl -TRAFFICGEN_IXNET_TCL_SCRIPT = 'ixnetrfc2544v2.tcl' - # VXLAN traffic item VXLAN_VNI = '99' diff --git a/core/traffic_controller.py b/core/traffic_controller.py index a6c1bd8d..b1911536 100644 --- a/core/traffic_controller.py +++ b/core/traffic_controller.py @@ -21,7 +21,6 @@ import time from core.results.results_constants import ResultsConstants from conf import settings -from conf import get_test_param class TrafficController(object): """Base class which defines a common functionality for all traffic @@ -41,21 +40,13 @@ class TrafficController(object): self._traffic_gen_class = traffic_gen_class() self._traffic_started = False self._traffic_started_call_count = 0 - self._duration = int(get_test_param('duration', 30)) - self._lossrate = float(get_test_param('lossrate', 0.0)) + self._duration = int(settings.getValue('TRAFFICGEN_DURATION')) + self._lossrate = float(settings.getValue('TRAFFICGEN_LOSSRATE')) + self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES') + self._mode = settings.getValue('mode').lower() self._results = [] - # If set, comma separated packet_sizes value from --test_params - # on cli takes precedence over value in settings file. - self._packet_sizes = None - packet_sizes_cli = get_test_param('pkt_sizes') - if packet_sizes_cli: - self._packet_sizes = [int(x.strip()) - for x in packet_sizes_cli.split(',')] - else: - self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES') - def __enter__(self): """Call initialisation function. """ diff --git a/core/traffic_controller_rfc2544.py b/core/traffic_controller_rfc2544.py index 874c3ae7..980205f6 100644 --- a/core/traffic_controller_rfc2544.py +++ b/core/traffic_controller_rfc2544.py @@ -15,7 +15,7 @@ """ from core.traffic_controller import TrafficController from core.results.results import IResults -from conf import get_test_param +from conf import settings class TrafficControllerRFC2544(TrafficController, IResults): @@ -32,7 +32,7 @@ class TrafficControllerRFC2544(TrafficController, IResults): """ super(TrafficControllerRFC2544, self).__init__(traffic_gen_class) self._type = 'rfc2544' - self._tests = int(get_test_param('rfc2544_tests', 1)) + self._tests = int(settings.getValue('TRAFFICGEN_RFC2544_TESTS')) def send_traffic(self, traffic): """See TrafficController for description @@ -85,4 +85,3 @@ class TrafficControllerRFC2544(TrafficController, IResults): result = self._traffic_gen_class.wait_rfc2544_throughput() result = self._append_results(result, packet_size) self._results.append(result) - diff --git a/core/traffic_controller_rfc2889.py b/core/traffic_controller_rfc2889.py index a2e12e6d..210d5f5f 100644 --- a/core/traffic_controller_rfc2889.py +++ b/core/traffic_controller_rfc2889.py @@ -15,7 +15,7 @@ """ from core.traffic_controller import TrafficController from core.results.results import IResults -from conf import get_test_param +from conf import settings class TrafficControllerRFC2889(TrafficController, IResults): @@ -32,7 +32,7 @@ class TrafficControllerRFC2889(TrafficController, IResults): """ super(TrafficControllerRFC2889, self).__init__(traffic_gen_class) self._type = 'rfc2889' - self._trials = int(get_test_param('rfc2889_trials', 1)) + self._trials = int(settings.getValue('TRAFFICGEN_RFC2889_TRIALS')) def send_traffic(self, traffic): """See TrafficController for description @@ -83,7 +83,6 @@ class TrafficControllerRFC2889(TrafficController, IResults): else: function['function']() result = self._traffic_gen_class.wait_rfc2889_forwarding( - traffic, trials=self._trials, duration=self._duration) + traffic, trials=self._trials, duration=self._duration) result = self._append_results(result, packet_size) self._results.append(result) - diff --git a/docs/configguide/trafficgen.rst b/docs/configguide/trafficgen.rst index 85fc35b8..efcc4d83 100644 --- a/docs/configguide/trafficgen.rst +++ b/docs/configguide/trafficgen.rst @@ -69,7 +69,7 @@ OR from the commandline: .. code-block:: console - $ ./vsperf --test-params "pkt_sizes=x,y" $TESTNAME + $ ./vsperf --test-params "TRAFFICGEN_PKT_SIZES=(x,y)" $TESTNAME You can also modify the traffic transmission duration and the number of tests run by the traffic generator by extending the example @@ -77,7 +77,8 @@ commandline above to: .. code-block:: console - $ ./vsperf --test-params "pkt_sizes=x,y;duration=10;rfc2544_tests=1" $TESTNAME + $ ./vsperf --test-params "TRAFFICGEN_PKT_SIZES=(x,y);TRAFFICGEN_DURATION=10;" \ + "TRAFFICGEN_RFC2544_TESTS=1" $TESTNAME Dummy Setup ------------ diff --git a/docs/design/vswitchperf_design.rst b/docs/design/vswitchperf_design.rst index 375fa12e..88b6ba88 100755 --- a/docs/design/vswitchperf_design.rst +++ b/docs/design/vswitchperf_design.rst @@ -53,7 +53,8 @@ for development purposes: .. code-block:: console - $ ./vsperf --test-params 'duration=10;rfc2544_tests=1;pkt_sizes=64' --tests 'pvp_tput' + $ ./vsperf --test-params 'TRAFFICGEN_DURATION=10;TRAFFICGEN_RFC2544_TESTS=1;' \ + 'TRAFFICGEN_PKT_SIZES=(64,)' pvp_tput Typical Test Sequence ===================== diff --git a/docs/requirements/vswitchperf_ltp.rst b/docs/requirements/vswitchperf_ltp.rst index 6b2ee9bc..14303de5 100644 --- a/docs/requirements/vswitchperf_ltp.rst +++ b/docs/requirements/vswitchperf_ltp.rst @@ -1304,7 +1304,7 @@ vsperf CI jobs are broken down into: * Runs everyday takes about 10 hours to complete. * TESTCASES_DAILY='phy2phy_tput back2back phy2phy_tput_mod_vlan phy2phy_scalability pvp_tput pvp_back2back pvvp_tput pvvp_back2back'. - * TESTPARAM_DAILY='--test-params pkt_sizes=64,128,512,1024,1518'. + * TESTPARAM_DAILY='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'. * Merge job: diff --git a/docs/userguide/integration.rst b/docs/userguide/integration.rst index 5b26a716..003e8adb 100755 --- a/docs/userguide/integration.rst +++ b/docs/userguide/integration.rst @@ -81,21 +81,21 @@ To run VXLAN encapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --integration + ./vsperf --conf-file user_settings.py --integration \ --test-params 'tunnel_type=vxlan' overlay_p2p_tput To run GRE encapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --integration + ./vsperf --conf-file user_settings.py --integration \ --test-params 'tunnel_type=gre' overlay_p2p_tput To run GENEVE encapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --integration + ./vsperf --conf-file user_settings.py --integration \ --test-params 'tunnel_type=geneve' overlay_p2p_tput To run OVS NATIVE tunnel tests (VXLAN/GRE/GENEVE): @@ -127,7 +127,7 @@ To run OVS NATIVE tunnel tests (VXLAN/GRE/GENEVE): .. code-block:: console - ./vsperf --conf-file user_settings.py --integration + ./vsperf --conf-file user_settings.py --integration \ --test-params 'tunnel_type=vxlan' overlay_p2p_tput @@ -189,7 +189,7 @@ To run GRE decapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --test-params 'tunnel_type=gre' + ./vsperf --conf-file user_settings.py --test-params 'tunnel_type=gre' \ --integration overlay_p2p_decap_cont @@ -246,7 +246,7 @@ To run GENEVE decapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --test-params 'tunnel_type=geneve' + ./vsperf --conf-file user_settings.py --test-params 'tunnel_type=geneve' \ --integration overlay_p2p_decap_cont @@ -330,7 +330,7 @@ To run VXLAN decapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --integration + ./vsperf --conf-file user_settings.py --integration \ --test-params 'tunnel_type=vxlan' overlay_p2p_decap_cont Executing Native/Vanilla OVS GRE decapsulation tests @@ -389,7 +389,7 @@ To run GRE decapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --integration + ./vsperf --conf-file user_settings.py --integration \ --test-params 'tunnel_type=gre' overlay_p2p_decap_cont Executing Native/Vanilla OVS GENEVE decapsulation tests @@ -448,7 +448,7 @@ To run GENEVE decapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --integration + ./vsperf --conf-file user_settings.py --integration \ --test-params 'tunnel_type=geneve' overlay_p2p_decap_cont @@ -498,5 +498,5 @@ To run VXLAN encapsulation+decapsulation tests: .. code-block:: console - ./vsperf --conf-file user_settings.py --integration + ./vsperf --conf-file user_settings.py --integration \ overlay_p2p_mod_tput diff --git a/docs/userguide/teststeps.rst b/docs/userguide/teststeps.rst index 51e62020..65a25b0a 100644 --- a/docs/userguide/teststeps.rst +++ b/docs/userguide/teststeps.rst @@ -523,8 +523,8 @@ To run the test: .. code-block:: console - ./vsperf --conf-file user_settings.py --test-params - "guest_loopback=linux_bridge" --integration ex_pvp_rule_l3da + ./vsperf --conf-file user_settings.py --test-params \ + "GUEST_LOOPBACK=['linux_bridge']" --integration ex_pvp_rule_l3da Forward packets based on UDP port ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/userguide/testusage.rst b/docs/userguide/testusage.rst index 97c74721..f446f261 100755 --- a/docs/userguide/testusage.rst +++ b/docs/userguide/testusage.rst @@ -105,14 +105,61 @@ or via another command line argument will override both the default and your custom configuration files. This "priority hierarchy" can be described like so (1 = max priority): -1. Command line arguments -2. Environment variables -3. Configuration file(s) +1. Testcase definition section ``Parameters`` +2. Command line arguments +3. Environment variables +4. Configuration file(s) Further details about configuration files evaluation and special behaviour of options with ``GUEST_`` prefix could be found at `design document `__. +Overriding values defined in configuration files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The configuration items can be overridden by command line argument +``--test-params``. In this case, the configuration items and +their values should be passed in form of ``item=value`` and separated +by semicolon. + +Example: + +.. code:: console + + $ ./vsperf --test-params "TRAFFICGEN_DURATION=10;TRAFFICGEN_PKT_SIZES=(128,);" \ + "GUEST_LOOPBACK=['testpmd','l2fwd']" pvvp_tput + +The second option is to override configuration items by ``Parameters`` section +of the test case definition. The configuration items can be added into ``Parameters`` +dictionary with their new values. These values will override values defined in +configuration files or specified by ``--test-params`` command line argument. + +Example: + +.. code:: python + + "Parameters" : {'TRAFFICGEN_PKT_SIZES' : (128,), + 'TRAFFICGEN_DURATION' : 10, + 'GUEST_LOOPBACK' : ['testpmd','l2fwd'], + } + +**NOTE:** In both cases, configuration item names and their values must be specified +in the same form as they are defined inside configuration files. Parameter names +must be specified in uppercase and data types of original and new value must match. +Python syntax rules related to data types and structures must be followed. +For example, parameter ``TRAFFICGEN_PKT_SIZES`` above is defined as a tuple +with a single value ``128``. In this case trailing comma is mandatory, otherwise +value can be wrongly interpreted as a number instead of a tuple and vsperf +execution would fail. Please check configuration files for default values and their +types and use them as a basis for any customized values. In case of any doubt, please +check official python documentation related to data structures like tuples, lists +and dictionaries. + +**NOTE:** Vsperf execution will terminate with runtime error in case, that unknown +parameter name is passed via ``--test-params`` CLI argument or defined in ``Parameters`` +section of test case definition. It is also forbidden to redefine a value of +``TEST_PARAMS`` configuration item via CLI or ``Parameters`` section. + vloop_vnf ^^^^^^^^^ @@ -192,9 +239,9 @@ Some tests allow for configurable parameters, including test duration .. code:: bash - $ ./vsperf --conf-file user_settings.py - --tests RFC2544Tput - --test-params "duration=10;pkt_sizes=128" + $ ./vsperf --conf-file user_settings.py \ + --tests RFC2544Tput \ + --test-params "TRAFFICGEN_DURATION=10;TRAFFICGEN_PKT_SIZES=(128,)" For all available options, check out the help dialog: @@ -283,9 +330,11 @@ To run tests using Vanilla OVS: .. code-block:: console - $ ./vsperf --conf-file=/10_custom.conf - --test-params "vanilla_tgen_tx_ip=n.n.n.n; - vanilla_tgen_tx_mac=nn:nn:nn:nn:nn:nn" + $ ./vsperf --conf-file=/10_custom.conf \ + --test-params "VANILLA_TGEN_PORT1_IP=n.n.n.n;" \ + "VANILLA_TGEN_PORT1_MAC=nn:nn:nn:nn:nn:nn;" \ + "VANILLA_TGEN_PORT2_IP=n.n.n.n;" \ + "VANILLA_TGEN_PORT2_MAC=nn:nn:nn:nn:nn:nn" 2. If needed, recompile src for all OVS variants @@ -389,7 +438,7 @@ Execution of test with PCI passthrough with vswitch disabled: .. code-block:: console - $ ./vsperf --conf-file=/10_custom.conf + $ ./vsperf --conf-file=/10_custom.conf \ --vswitch none --vnf QemuPciPassthrough pvp_tput Any of supported guest-loopback-application_ can be used inside VM with @@ -414,8 +463,8 @@ or use ``--test-params`` CLI argument: .. code-block:: console - $ ./vsperf --conf-file=/10_custom.conf - --test-params "guest_loopback=testpmd" + $ ./vsperf --conf-file=/10_custom.conf \ + --test-params "GUEST_LOOPBACK=['testpmd']" Supported loopback applications are: @@ -549,15 +598,6 @@ environment. on the same numa as the NIC in use if possible/applicable. Testpmd should be assigned at least (nb_cores +1) total cores with the cpu mask. -The following CLI parameters override the corresponding configuration settings: - -1. ``guest_nic_queues``, which overrides all ``GUEST_NIC_QUEUES`` values -2. ``guest_testpmd_params``, which overrides all ``GUEST_TESTPMD_PARAMS`` - values -3. ``vswitch_dpdk_multi_queues``, which overrides ``VSWITCH_DPDK_MULTI_QUEUES`` -4. ``guest_smp``, which overrides all ``GUEST_SMP`` values -5. ``guest_core_binding``, which overrides all ``GUEST_CORE_BINDING`` values - Executing Packet Forwarding tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -573,8 +613,8 @@ or use ``--vswitch`` and ``--fwdapp`` CLI arguments: .. code-block:: console - $ ./vsperf --conf-file user_settings.py - --vswitch none + $ ./vsperf --conf-file user_settings.py \ + --vswitch none \ --fwdapp TestPMD Supported Packet Forwarding applications are: @@ -629,7 +669,7 @@ Mode of operation is driven by configuration parameter -m or --mode "trafficgen-pause" - execute vSwitch and VNF but wait before traffic transmission In case, that VSPERF is executed in "trafficgen" mode, then configuration -of traffic generator should be configured through --test-params option. +of traffic generator should be configured through ``--test-params`` option. Supported CLI options useful for traffic generator configuration are: .. code-block:: console @@ -656,7 +696,7 @@ Example of execution of VSPERF in "trafficgen" mode: .. code-block:: console - $ ./vsperf -m trafficgen --trafficgen IxNet --conf-file vsperf.conf + $ ./vsperf -m trafficgen --trafficgen IxNet --conf-file vsperf.conf \ --test-params "traffic_type=continuous;bidirectional=True;iload=60" Code change verification by pylint diff --git a/testcases/testcase.py b/testcases/testcase.py index 00164ea2..01a07391 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -66,6 +66,7 @@ class TestCase(object): self._step_vnf_list = {} self._step_result = [] self._step_status = None + self._testcase_run_time = None # store all GUEST_ specific settings to keep original values before their expansion for key in S.__dict__: @@ -75,14 +76,21 @@ class TestCase(object): self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH'))) self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF'))) self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN'))) - self._update_settings('TEST_PARAMS', cfg.get('Parameters', S.getValue('TEST_PARAMS'))) + test_params = copy.deepcopy(S.getValue('TEST_PARAMS')) + tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS')) + test_params.update(tc_test_params) + self._update_settings('TEST_PARAMS', test_params) + S.check_test_params() + + # override all redefined GUEST_ values to have them expanded correctly + tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS')) + for key in tmp_test_params: + if key.startswith('GUEST_'): + S.setValue(key, S.getValue(key)) + S.getValue('TEST_PARAMS').pop(key) # update global settings functions.settings_update_paths() - guest_loopback = get_test_param('guest_loopback', None) - if guest_loopback: - # we can put just one item, it'll be expanded automatically for all VMs - self._update_settings('GUEST_LOOPBACK', [guest_loopback]) # set test parameters; CLI options take precedence to testcase settings self._logger = logging.getLogger(__name__) @@ -337,7 +345,8 @@ class TestCase(object): self.run_finalize() self._testcase_run_time = time.strftime("%H:%M:%S", - time.gmtime(time.time() - self._testcase_start_time)) + time.gmtime(time.time() - + self._testcase_start_time)) logging.info("Testcase execution time: " + self._testcase_run_time) # report test results self.run_report() @@ -352,7 +361,7 @@ class TestCase(object): """ orig_value = S.getValue(param) if orig_value != value: - self._settings_original[param] = orig_value + self._settings_original[param] = copy.deepcopy(orig_value) S.setValue(param, value) def _append_results(self, results): diff --git a/tools/pkt_gen/ixnet/ixnet.py b/tools/pkt_gen/ixnet/ixnet.py index 52ba1712..6262a10a 100755 --- a/tools/pkt_gen/ixnet/ixnet.py +++ b/tools/pkt_gen/ixnet/ixnet.py @@ -151,13 +151,17 @@ class IxNet(trafficgen.ITrafficGenerator): Currently only the RFC2544 tests are implemented. """ - _script = os.path.join(settings.getValue('TRAFFICGEN_IXIA_3RD_PARTY'), - settings.getValue('TRAFFICGEN_IXNET_TCL_SCRIPT')) - _tclsh = tkinter.Tcl() - _cfg = None - _logger = logging.getLogger(__name__) - _params = None - _bidir = None + + def __init__(self): + """Initialize IXNET members + """ + self._script = os.path.join(settings.getValue('TRAFFICGEN_IXIA_3RD_PARTY'), + settings.getValue('TRAFFICGEN_IXNET_TCL_SCRIPT')) + self._tclsh = tkinter.Tcl() + self._cfg = None + self._logger = logging.getLogger(__name__) + self._params = None + self._bidir = None def run_tcl(self, cmd): """Run a TCL script using the TCL interpreter found in ``tkinter``. diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py index 51553277..997f93e0 100644 --- a/vnfs/qemu/qemu.py +++ b/vnfs/qemu/qemu.py @@ -24,7 +24,6 @@ import time import pexpect from conf import settings as S -from conf import get_test_param from vnfs.vnf.vnf import IVnf class IVnfQemu(IVnf): @@ -69,7 +68,6 @@ class IVnfQemu(IVnf): self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr] # set guest loopback application based on VNF configuration - # cli option take precedence to config file values self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number] self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number] @@ -81,11 +79,6 @@ class IVnfQemu(IVnf): self._testpmd_fwd_mode, 'io') self._testpmd_fwd_mode = 'io' - guest_smp = int(get_test_param('guest_smp', 0)) - if guest_smp: - override_list = [guest_smp] * (self._number + 1) - S.setValue('GUEST_SMP', override_list) - name = 'Client%d' % self._number vnc = ':%d' % self._number # NOTE: affinization of main qemu process can cause hangup of 2nd VM @@ -238,18 +231,14 @@ class IVnfQemu(IVnf): stdin=proc.stdout) proc.wait() - guest_core_binding = int(get_test_param('guest_core_binding', 0)) for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])): match = None for line in output.decode(cur_locale).split('\n'): match = re.search(thread_id % cpu, line) if match: - if guest_core_binding: - self._affinitize_pid(guest_core_binding, match.group(1)) - else: - self._affinitize_pid( - S.getValue('GUEST_CORE_BINDING')[self._number][cpu], - match.group(1)) + self._affinitize_pid( + S.getValue('GUEST_CORE_BINDING')[self._number][cpu], + match.group(1)) break if not match: @@ -397,10 +386,9 @@ class IVnfQemu(IVnf): self.execute_and_wait('make') # get testpmd settings from CLI - testpmd_params = get_test_param('guest_testpmd_params', - S.getValue('GUEST_TESTPMD_PARAMS')[self._number]) + testpmd_params = S.getValue('GUEST_TESTPMD_PARAMS')[self._number] - self.execute_and_wait( './testpmd {}'.format(testpmd_params), 60, "Done") + self.execute_and_wait('./testpmd {}'.format(testpmd_params), 60, "Done") self.execute('set fwd ' + self._testpmd_fwd_mode, 1) self.execute_and_wait('start', 20, 'testpmd>') @@ -458,17 +446,13 @@ class IVnfQemu(IVnf): # Add the arp entries for the IXIA ports and the bridge you are using. # Use command line values if provided. - trafficgen_mac = get_test_param('vanilla_tgen_port1_mac', - S.getValue('VANILLA_TGEN_PORT1_MAC')) - trafficgen_ip = get_test_param('vanilla_tgen_port1_ip', - S.getValue('VANILLA_TGEN_PORT1_IP')) + trafficgen_mac = S.getValue('VANILLA_TGEN_PORT1_MAC') + trafficgen_ip = S.getValue('VANILLA_TGEN_PORT1_IP') self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) - trafficgen_mac = get_test_param('vanilla_tgen_port2_mac', - S.getValue('VANILLA_TGEN_PORT2_MAC')) - trafficgen_ip = get_test_param('vanilla_tgen_port2_ip', - S.getValue('VANILLA_TGEN_PORT2_IP')) + trafficgen_mac = S.getValue('VANILLA_TGEN_PORT2_MAC') + trafficgen_ip = S.getValue('VANILLA_TGEN_PORT2_IP') self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) diff --git a/vnfs/qemu/qemu_dpdk_vhost_user.py b/vnfs/qemu/qemu_dpdk_vhost_user.py index a25c61ef..51c10242 100644 --- a/vnfs/qemu/qemu_dpdk_vhost_user.py +++ b/vnfs/qemu/qemu_dpdk_vhost_user.py @@ -18,7 +18,6 @@ import logging from conf import settings as S -from conf import get_test_param from vnfs.qemu.qemu import IVnfQemu class QemuDpdkVhostUser(IVnfQemu): @@ -33,10 +32,6 @@ class QemuDpdkVhostUser(IVnfQemu): self._logger = logging.getLogger(__name__) # multi-queue values - guest_nic_queues = int(get_test_param('guest_nic_queues', 0)) - if guest_nic_queues: - override_list = [guest_nic_queues] * (self._number + 1) - S.setValue('GUEST_NIC_QUEUES', override_list) if int(S.getValue('GUEST_NIC_QUEUES')[self._number]): queue_str = ',queues={}'.format(S.getValue('GUEST_NIC_QUEUES')[self._number]) mq_vector_str = ',mq=on,vectors={}'.format( diff --git a/vsperf b/vsperf index 5bfa2abe..ee494dcd 100755 --- a/vsperf +++ b/vsperf @@ -21,6 +21,7 @@ import logging import os import sys import argparse +import re import time import datetime import shutil @@ -75,21 +76,19 @@ def parse_arguments(): This expects either 'x=y', 'x=y,z' or 'x' (implicit true) values. For multiple overrides use a ; separated list for - e.g. --test-params 'x=z; y=a,b' + e.g. --test-params 'x=z; y=(a,b)' """ def __call__(self, parser, namespace, values, option_string=None): results = {} - for value in values.split(';'): - result = [key.strip() for key in value.split('=')] - if len(result) == 1: - results[result[0]] = True - elif len(result) == 2: - results[result[0]] = result[1] - else: - raise argparse.ArgumentTypeError( - 'expected \'%s\' to be of format \'key=val\' or' - ' \'key\'' % result) + for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values): + param = param.strip() + value = value.strip() + if len(param): + if len(value): + results[param] = value + else: + results[param] = True setattr(namespace, self.dest, results) @@ -181,9 +180,9 @@ def parse_arguments(): group.add_argument('--conf-file', action=_ValidateFileAction, help='settings file') group.add_argument('--test-params', action=_SplitTestParamsAction, - help='csv list of test parameters: key=val; e.g.' - 'including pkt_sizes=x,y; duration=x; ' - 'rfc2544_tests=x ...') + help='csv list of test parameters: key=val; e.g. ' + 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFICGEN_DURATION=30; ' + 'GUEST_LOOPBACK=["l2fwd"] ...') group.add_argument('--opnfvpod', help='name of POD in opnfv') args = vars(parser.parse_args()) @@ -585,14 +584,6 @@ def main(): # for backward compatibility settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list)) - # update global settings - guest_loopback = get_test_param('guest_loopback', None) - if guest_loopback: - tmp_gl = [] - for dummy_i in range(len(settings.getValue('GUEST_LOOPBACK'))): - tmp_gl.append(guest_loopback) - settings.setValue('GUEST_LOOPBACK', tmp_gl) - settings.setValue('mode', args['mode']) # generate results directory name @@ -612,11 +603,11 @@ def main(): loader = Loader() # set traffic details, so they can be passed to traffic ctl traffic = copy.deepcopy(TRAFFIC_DEFAULTS) - traffic.update({'traffic_type': get_test_param('traffic_type', 'rfc2544'), - 'bidir': get_test_param('bidirectional', 'False'), - 'multistream': int(get_test_param('multistream', 0)), - 'stream_type': get_test_param('stream_type', 'L4'), - 'frame_rate': int(get_test_param('iload', 100))}) + traffic.update({'traffic_type': get_test_param('traffic_type', TRAFFIC_DEFAULTS['traffic_type']), + 'bidir': get_test_param('bidirectional', TRAFFIC_DEFAULTS['bidir']), + 'multistream': int(get_test_param('multistream', TRAFFIC_DEFAULTS['multistream'])), + 'stream_type': get_test_param('stream_type', TRAFFIC_DEFAULTS['stream_type']), + 'frame_rate': int(get_test_param('iload', TRAFFIC_DEFAULTS['frame_rate']))}) traffic_ctl = component_factory.create_traffic( traffic['traffic_type'], diff --git a/vswitches/ovs_dpdk_vhost.py b/vswitches/ovs_dpdk_vhost.py index 5beb0051..67011d20 100644 --- a/vswitches/ovs_dpdk_vhost.py +++ b/vswitches/ovs_dpdk_vhost.py @@ -17,12 +17,10 @@ import logging import subprocess -import os from src.ovs import OFBridge from src.dpdk import dpdk from conf import settings -from conf import get_test_param from vswitches.ovs import IVSwitchOvs class OvsDpdkVhost(IVSwitchOvs): @@ -70,11 +68,6 @@ class OvsDpdkVhost(IVSwitchOvs): dpdk.init() super(OvsDpdkVhost, self).start() # old style OVS <= 2.5.0 multi-queue enable - vswitch_dpdk_multi_queues = \ - int(get_test_param('vswitch_dpdk_multi_queues', 0)) - if vswitch_dpdk_multi_queues: - settings.setValue('VSWITCH_DPDK_MULTI_QUEUES', \ - vswitch_dpdk_multi_queues) if settings.getValue('OVS_OLD_STYLE_MQ') and \ int(settings.getValue('VSWITCH_DPDK_MULTI_QUEUES')): tmp_br = OFBridge(timeout=-1) @@ -120,8 +113,8 @@ class OvsDpdkVhost(IVSwitchOvs): if int(settings.getValue('VSWITCH_DPDK_MULTI_QUEUES')) and \ not settings.getValue('OVS_OLD_STYLE_MQ'): - params += ['options:n_rxq={}'.format( - settings.getValue('VSWITCH_DPDK_MULTI_QUEUES'))] + params += ['options:n_rxq={}'.format( + settings.getValue('VSWITCH_DPDK_MULTI_QUEUES'))] of_port = bridge.add_port(port_name, params) return (port_name, of_port) -- cgit 1.2.3-korg