From b1534957e463b5e34957a8d48ce5c6b0552ffbb4 Mon Sep 17 00:00:00 2001 From: Martin Klozik Date: Thu, 31 Aug 2017 15:01:18 +0200 Subject: teststeps: Improvements and bugfixing of teststeps This patch introduces several improvements and small bugfixes of teststeps. These changes were identified during implementation of OVS/DPDK regression tests. Patch content: * teststeps: step aliases were implemented * teststeps: improved filtering by regex for any step, which returns string or list of stings; filter will process all lines * teststeps: support for log object * teststeps: support for trafficgen get_results call * teststeps: configurable suppression of step validation * trafficgen: remove old results before traffic is executed * trafficgen: support for flow control on/off (IxNet) * trafficgen: support for configurable learning frames (IxNet) * trafficgen: support for runtime changes of TRAFFICGEN_PKT_SIZES, _DURATION and _LOSSRATE * vnf: flush pexpect output of previous commands * vnf: use execute_and_wait() to ensure correct cmds order * vnf: dpdk vHost User interface name set according to its type, e.g. dpdkvhostuserclient * vswitch: support for OVS restart * decap: simplify configuration of tunneling decapsulation tests * settings: values of all configuration options are restored after TC execution * modified formatting of test description used by --list * testcase name and description is logged before its execution * small bugfixes JIRA: VSPERF-539 Change-Id: I550ba0d897ece89abd3f33d6d66f545c4d863e7b Signed-off-by: Martin Klozik Reviewed-by: Al Morton Reviewed-by: Christian Trautman Reviewed-by: Sridhar Rao Reviewed-by: Trevor Cooper --- 3rd_party/ixia/ixnetrfc2544.tcl | 11 +- conf/03_traffic.conf | 15 ++- conf/04_vnf.conf | 2 + conf/__init__.py | 12 +++ conf/integration/02_vswitch.conf | 6 -- conf/integration/03_traffic.conf | 6 +- core/traffic_controller.py | 39 +++++-- core/traffic_controller_rfc2544.py | 18 ++-- core/traffic_controller_rfc2889.py | 18 ++-- .../devguide/design/vswitchperf_design.rst | 12 +++ docs/testing/user/userguide/integration.rst | 39 ++----- docs/testing/user/userguide/teststeps.rst | 88 ++++++++++++++-- testcases/testcase.py | 114 +++++++++++++-------- tools/functions.py | 42 ++++++++ tools/pkt_gen/ixnet/ixnet.py | 11 +- tools/teststepstools.py | 8 +- vnfs/qemu/qemu.py | 96 ++++++++--------- vnfs/qemu/qemu_dpdk_vhost_user.py | 9 +- vnfs/vnf/vnf.py | 60 +++++++---- vsperf | 39 +++++-- vswitches/ovs.py | 27 +++++ vswitches/ovs_dpdk_vhost.py | 2 +- vswitches/vswitch.py | 7 ++ 23 files changed, 469 insertions(+), 212 deletions(-) diff --git a/3rd_party/ixia/ixnetrfc2544.tcl b/3rd_party/ixia/ixnetrfc2544.tcl index 73e6d780..c47e8fc1 100644 --- a/3rd_party/ixia/ixnetrfc2544.tcl +++ b/3rd_party/ixia/ixnetrfc2544.tcl @@ -113,7 +113,7 @@ proc startRfc2544Test { testSpec trafficSpec } { set loadType custom } - set learningFrames True + set learningFrames [dict get $testSpec learningFrames] set L2CountValue 1 set L2Increment False @@ -150,6 +150,7 @@ proc startRfc2544Test { testSpec trafficSpec } { } } + set flowControl [dict get $testSpec flowControl] set fastConvergence True set convergenceDuration [expr $duration/10] @@ -403,7 +404,7 @@ proc startRfc2544Test { testSpec trafficSpec } { -txIgnoreRxLinkFaults False \ -loopback False \ -enableLASIMonitoring False \ - -enabledFlowControl True + -enabledFlowControl $flowControl ixNet setMultiAttrs $sg_vport/l1Config/tenGigLan/oam \ -tlvType {00} \ -linkEvents False \ @@ -432,7 +433,7 @@ proc startRfc2544Test { testSpec trafficSpec } { -txIgnoreRxLinkFaults False \ -loopback False \ -enableLASIMonitoring False \ - -enabledFlowControl False + -enabledFlowControl $flowControl ixNet setMultiAttrs $sg_vport/l1Config/fortyGigLan/fcoe \ -supportDataCenterMode False \ -priorityGroupSize priorityGroupSize-8 \ @@ -784,7 +785,7 @@ proc startRfc2544Test { testSpec trafficSpec } { -txIgnoreRxLinkFaults False \ -loopback False \ -enableLASIMonitoring False \ - -enabledFlowControl False + -enabledFlowControl $flowControl ixNet setMultiAttrs $sg_vport/l1Config/tenGigLan/oam \ -tlvType {00} \ -linkEvents False \ @@ -813,7 +814,7 @@ proc startRfc2544Test { testSpec trafficSpec } { -txIgnoreRxLinkFaults False \ -loopback False \ -enableLASIMonitoring False \ - -enabledFlowControl False + -enabledFlowControl $flowControl ixNet setMultiAttrs $sg_vport/l1Config/fortyGigLan/fcoe \ -supportDataCenterMode False \ -priorityGroupSize priorityGroupSize-8 \ diff --git a/conf/03_traffic.conf b/conf/03_traffic.conf index b5533833..6cfbbf23 100644 --- a/conf/03_traffic.conf +++ b/conf/03_traffic.conf @@ -71,6 +71,18 @@ LOG_FILE_TRAFFIC_GEN = 'traffic-gen.log' # "IP" - flow is defined by ingress ports # and src and dst IP addresses # Default value: "port" +# 'flow_control' - Controls flow control support by traffic generator. +# Supported values: +# False - flow control is disabled +# True - flow control is enabled +# Default value: False +# Note: Currently it is supported by IxNet only +# 'learning_frames' - Controls learning frames support by traffic generator. +# Supported values: +# False - learning freames are disabled +# True - learning freames are enabled +# Default value: True +# Note: Currently it is supported by IxNet only # 'l2' - A dictionary with l2 network layer details. Supported # values are: # 'srcmac' - Specifies source MAC address filled by traffic generator. @@ -143,7 +155,8 @@ TRAFFIC = { 'stream_type' : 'L4', 'pre_installed_flows' : 'No', # used by vswitch implementation 'flow_type' : 'port', # used by vswitch implementation - + 'flow_control' : False, # supported only by IxNet + 'learning_frames' : True, # supported only by IxNet 'l2': { 'framesize': 64, 'srcmac': '00:00:00:00:00:00', diff --git a/conf/04_vnf.conf b/conf/04_vnf.conf index eafec74b..37fbe2b1 100644 --- a/conf/04_vnf.conf +++ b/conf/04_vnf.conf @@ -96,6 +96,8 @@ GUEST_SHARED_DRIVE_TYPE = ['scsi'] # 'linux_bridge' - linux bridge will be configured # 'buildin' - nothing will be configured by vsperf; VM image must # ensure traffic forwarding between its interfaces +# 'clean' - nothing will be configured, but automatic login will +# be performed; Useful for stepdriven testcases. # For 2 VNFs you may use ['testpmd', 'l2fwd'] GUEST_LOOPBACK = ['testpmd'] diff --git a/conf/__init__.py b/conf/__init__.py index 808cfc97..a7c0ee5d 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -189,6 +189,18 @@ class Settings(object): else: setattr(self, key.upper(), conf[key]) + def restore_from_dict(self, conf): + """ + Restore ``settings`` with values found in ``conf``. + + Method will drop all configuration options and restore their + values from conf dictionary + """ + self.__dict__.clear() + tmp_conf = copy.deepcopy(conf) + for key in tmp_conf: + self.setValue(key, tmp_conf[key]) + def load_from_env(self): """ Update ``settings`` with values found in the environment. diff --git a/conf/integration/02_vswitch.conf b/conf/integration/02_vswitch.conf index 68eaf73e..63ffe1bc 100644 --- a/conf/integration/02_vswitch.conf +++ b/conf/integration/02_vswitch.conf @@ -23,12 +23,6 @@ TUNNEL_EXTERNAL_BRIDGE_IP = '192.168.240.1/24' # vxlan|gre|geneve TUNNEL_TYPE = 'vxlan' -# The receiving NIC of VXLAN traffic -# Used for OVS Vanilla Decap -DUT_NIC1_MAC = '' -# Used for OVS DPDK Decap -DUT_NIC2_MAC = '' - #Tunnel bridge configuration for P-TUN-P(VxLAN) deployment scenario # to test VxLAN performance without any overlay ingress traffic by doing the # encap decap inside the virtual switch itself. diff --git a/conf/integration/03_traffic.conf b/conf/integration/03_traffic.conf index e78e2668..5126f515 100644 --- a/conf/integration/03_traffic.conf +++ b/conf/integration/03_traffic.conf @@ -29,7 +29,7 @@ VXLAN_VNI = '99' # '00:1b:21:b3:48:a9'} VXLAN_FRAME_L2 = {'srcmac': '01:02:03:04:05:06', - 'dstmac': DUT_NIC2_MAC, + 'dstmac': '06:05:04:03:02:01', } # VXLAN is supported both in IxNetwork and IXIA IxExplorer @@ -71,7 +71,7 @@ VXLAN_FRAME_L4 = {'srcport': 4789, # TEST frame # dstmac should be set to the MAC address of the DUT's receiving port GRE_FRAME_L2 = {'srcmac': '01:02:03:04:05:06', - 'dstmac': DUT_NIC2_MAC, + 'dstmac': '06:05:04:03:02:01', } GRE_FRAME_L3 = {'proto': 'gre', @@ -95,7 +95,7 @@ GRE_FRAME_L4 = {'srcport': 0, # TEST frame # dstmac should be set to the MAC address of the DUT's receiving port GENEVE_FRAME_L2 = {'srcmac': '01:02:03:04:05:06', - 'dstmac': DUT_NIC2_MAC, + 'dstmac': '06:05:04:03:02:01', } GENEVE_FRAME_L3 = {'proto': 'udp', diff --git a/core/traffic_controller.py b/core/traffic_controller.py index 5ebdc0d9..d6e7629c 100644 --- a/core/traffic_controller.py +++ b/core/traffic_controller.py @@ -40,13 +40,25 @@ class TrafficController(object): self._traffic_gen_class = traffic_gen_class() self._traffic_started = False self._traffic_started_call_count = 0 + self._duration = None + self._lossrate = None + self._packet_sizes = None + + self._mode = str(settings.getValue('mode')).lower() + self._results = [] + + def configure(self, traffic): + """Set configuration values just before test execution so they + can be changed during runtime by test steps. + """ self._duration = int(settings.getValue('TRAFFICGEN_DURATION')) self._lossrate = float(settings.getValue('TRAFFICGEN_LOSSRATE')) self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES') - - self._mode = str(settings.getValue('mode')).lower() self._results = [] + # update type with detailed traffic value + self._type = traffic['traffic_type'] + def __enter__(self): """Call initialisation function. """ @@ -101,18 +113,18 @@ class TrafficController(object): print("Please respond with 'yes', 'y', 'no' or 'n' ", end='') return True - def send_traffic(self, dummy_traffic): + def send_traffic(self, traffic): """Triggers traffic to be sent from the traffic generator. This is a blocking function. :param traffic: A dictionary describing the traffic to send. """ - raise NotImplementedError( - "The TrafficController does not implement", - "the \"send_traffic\" function.") + self._logger.debug('send_traffic with ' + + str(self._traffic_gen_class)) + self.configure(traffic) - def send_traffic_async(self, dummy_traffic, dummy_function): + def send_traffic_async(self, traffic, dummy_function): """Triggers traffic to be sent asynchronously. This is not a blocking function. @@ -127,9 +139,9 @@ class TrafficController(object): If this function requires more than one argument, all should be should be passed using the args list and appropriately handled. """ - raise NotImplementedError( - "The TrafficController does not implement", - "the \"send_traffic_async\" function.") + self._logger.debug('send_traffic_async with ' + + str(self._traffic_gen_class)) + self.configure(traffic) def stop_traffic(self): """Kills traffic being sent from the traffic generator. @@ -155,7 +167,7 @@ class TrafficController(object): def validate_send_traffic(self, dummy_result, dummy_traffic): """Verify that send traffic has succeeded """ - if len(self._results): + if self._results: if 'b2b_frames' in self._results[-1]: return float(self._results[-1]['b2b_frames']) > 0 elif 'throughput_rx_fps' in self._results[-1]: @@ -164,3 +176,8 @@ class TrafficController(object): return True else: return False + + def validate_get_results(self, result): + """Verify that results has been returned + """ + return self._results == result diff --git a/core/traffic_controller_rfc2544.py b/core/traffic_controller_rfc2544.py index cb839518..488dde6f 100644 --- a/core/traffic_controller_rfc2544.py +++ b/core/traffic_controller_rfc2544.py @@ -30,8 +30,14 @@ class TrafficControllerRFC2544(TrafficController, IResults): :param traffic_gen_class: The traffic generator class to be used. """ - super(TrafficControllerRFC2544, self).__init__(traffic_gen_class) + super().__init__(traffic_gen_class) self._type = 'rfc2544' + self._tests = None + + def configure(self, traffic): + """See TrafficController for description + """ + super().configure(traffic) self._tests = int(settings.getValue('TRAFFICGEN_RFC2544_TESTS')) def send_traffic(self, traffic): @@ -39,11 +45,8 @@ class TrafficControllerRFC2544(TrafficController, IResults): """ if not self.traffic_required(): return - self._logger.debug('send_traffic with ' + - str(self._traffic_gen_class)) - # update type with detailed traffic value - self._type = traffic['traffic_type'] + super().send_traffic(traffic) for packet_size in self._packet_sizes: # Merge framesize with the default traffic definition @@ -74,11 +77,8 @@ class TrafficControllerRFC2544(TrafficController, IResults): """ if not self.traffic_required(): return - self._logger.debug('send_traffic_async with ' + - str(self._traffic_gen_class)) - # update type with detailed traffic value - self._type = traffic['traffic_type'] + super().send_traffic_async(traffic, function) for packet_size in self._packet_sizes: traffic['l2'] = {'framesize': packet_size} diff --git a/core/traffic_controller_rfc2889.py b/core/traffic_controller_rfc2889.py index 01aaa722..64ab0ba6 100644 --- a/core/traffic_controller_rfc2889.py +++ b/core/traffic_controller_rfc2889.py @@ -30,8 +30,14 @@ class TrafficControllerRFC2889(TrafficController, IResults): :param traffic_gen_class: The traffic generator class to be used. """ - super(TrafficControllerRFC2889, self).__init__(traffic_gen_class) + super().__init__(traffic_gen_class) self._type = 'rfc2889' + self._trials = None + + def configure(self, traffic): + """See TrafficController for description + """ + super().configure(traffic) self._trials = int(settings.getValue('TRAFFICGEN_RFC2889_TRIALS')) def send_traffic(self, traffic): @@ -39,11 +45,8 @@ class TrafficControllerRFC2889(TrafficController, IResults): """ if not self.traffic_required(): return - self._logger.debug('send_traffic with ' + - str(self._traffic_gen_class)) - # update type with detailed traffic value - self._type = traffic['traffic_type'] + super().send_traffic(traffic) for packet_size in self._packet_sizes: # Merge framesize with the default traffic definition @@ -71,11 +74,8 @@ class TrafficControllerRFC2889(TrafficController, IResults): """ if not self.traffic_required(): return - self._logger.debug('send_traffic_async with ' + - str(self._traffic_gen_class)) - # update type with detailed traffic value - self._type = traffic['traffic_type'] + super().send_traffic_async(traffic, function) for packet_size in self._packet_sizes: traffic['l2'] = {'framesize': packet_size} diff --git a/docs/testing/developer/devguide/design/vswitchperf_design.rst b/docs/testing/developer/devguide/design/vswitchperf_design.rst index 55614d33..33051493 100644 --- a/docs/testing/developer/devguide/design/vswitchperf_design.rst +++ b/docs/testing/developer/devguide/design/vswitchperf_design.rst @@ -339,6 +339,18 @@ Detailed description of ``TRAFFIC`` dictionary items follows: "IP" - flow is defined by ingress ports and src and dst IP addresses Default value: "port" + 'flow_control' - Controls flow control support by traffic generator. + Supported values: + False - flow control is disabled + True - flow control is enabled + Default value: False + Note: Currently it is supported by IxNet only + 'learning_frames' - Controls learning frames support by traffic generator. + Supported values: + False - learning frames are disabled + True - learning frames are enabled + Default value: True + Note: Currently it is supported by IxNet only 'l2' - A dictionary with l2 network layer details. Supported values are: 'srcmac' - Specifies source MAC address filled by traffic generator. diff --git a/docs/testing/user/userguide/integration.rst b/docs/testing/user/userguide/integration.rst index 249a03c4..66808400 100644 --- a/docs/testing/user/userguide/integration.rst +++ b/docs/testing/user/userguide/integration.rst @@ -140,13 +140,7 @@ To run VXLAN decapsulation tests: 1. Set the variables used in "Executing Tunnel encapsulation tests" -2. Set dstmac of DUT_NIC2_MAC to the MAC adddress of the 2nd NIC of your DUT - - .. code-block:: python - - DUT_NIC2_MAC = '' - -3. Run test: +2. Run test: .. code-block:: console @@ -181,13 +175,7 @@ To run GRE decapsulation tests: 1. Set the variables used in "Executing Tunnel encapsulation tests" -2. Set dstmac of DUT_NIC2_MAC to the MAC adddress of the 2nd NIC of your DUT - - .. code-block:: python - - DUT_NIC2_MAC = '' - -3. Run test: +2. Run test: .. code-block:: console @@ -238,13 +226,7 @@ To run GENEVE decapsulation tests: 1. Set the variables used in "Executing Tunnel encapsulation tests" -2. Set dstmac of DUT_NIC2_MAC to the MAC adddress of the 2nd NIC of your DUT - - .. code-block:: python - - DUT_NIC2_MAC = '' - -3. Run test: +2. Run test: .. code-block:: console @@ -289,8 +271,6 @@ To run VXLAN decapsulation tests: 'datapath/linux/openvswitch.ko', ] - DUT_NIC1_MAC = '' - TRAFFICGEN_PORT1_IP = '172.16.1.2' TRAFFICGEN_PORT2_IP = '192.168.1.11' @@ -302,7 +282,8 @@ To run VXLAN decapsulation tests: VXLAN_FRAME_L2 = {'srcmac': '01:02:03:04:05:06', - 'dstmac': DUT_NIC1_MAC + 'dstmac': + '06:05:04:03:02:01', } VXLAN_FRAME_L3 = {'proto': 'udp', @@ -349,8 +330,6 @@ To run GRE decapsulation tests: 'datapath/linux/openvswitch.ko', ] - DUT_NIC1_MAC = '' - TRAFFICGEN_PORT1_IP = '172.16.1.2' TRAFFICGEN_PORT2_IP = '192.168.1.11' @@ -362,7 +341,8 @@ To run GRE decapsulation tests: GRE_FRAME_L2 = {'srcmac': '01:02:03:04:05:06', - 'dstmac': DUT_NIC1_MAC + 'dstmac': + '06:05:04:03:02:01', } GRE_FRAME_L3 = {'proto': 'udp', @@ -408,8 +388,6 @@ To run GENEVE decapsulation tests: 'datapath/linux/openvswitch.ko', ] - DUT_NIC1_MAC = '' - TRAFFICGEN_PORT1_IP = '172.16.1.2' TRAFFICGEN_PORT2_IP = '192.168.1.11' @@ -421,7 +399,8 @@ To run GENEVE decapsulation tests: GENEVE_FRAME_L2 = {'srcmac': '01:02:03:04:05:06', - 'dstmac': DUT_NIC1_MAC + 'dstmac': + '06:05:04:03:02:01', } GENEVE_FRAME_L3 = {'proto': 'udp', diff --git a/docs/testing/user/userguide/teststeps.rst b/docs/testing/user/userguide/teststeps.rst index 5349d2ea..8be67310 100644 --- a/docs/testing/user/userguide/teststeps.rst +++ b/docs/testing/user/userguide/teststeps.rst @@ -29,6 +29,14 @@ does not pass validation the test will fail and terminate. The test will continu until a failure is detected or all steps pass. A csv report file is generated after a test completes with an OK or FAIL result. +**NOTE**: It is possible to suppress validation process of given step by prefixing +it by ``!`` (exclamation mark). +In following example test execution won't fail if all traffic is dropped: + +.. code-block:: python + + ['!trafficgen', 'send_traffic', {}] + In case of performance test, the validation of steps is not performed and standard output files with results from traffic generator and underlying OS details are generated by vsperf. @@ -75,6 +83,7 @@ of supported objects and their most common functions follows: * ``disable_stp br_name`` - disables Spanning Tree Protocol for bridge ``br_name`` * ``enable_rstp br_name`` - enables Rapid Spanning Tree Protocol for bridge ``br_name`` * ``disable_rstp br_name`` - disables Rapid Spanning Tree Protocol for bridge ``br_name`` + * ``restart`` - restarts switch, which is useful for failover testcases Examples: @@ -101,15 +110,26 @@ of supported objects and their most common functions follows: * ``start`` - starts a VNF based on VSPERF configuration * ``stop`` - gracefully terminates given VNF + * ``execute command [delay]`` - executes command `cmd` inside VNF; Optional + delay defines number of seconds to wait before next step is executed. Method + returns command output as a string. + * ``execute_and_wait command [timeout] [prompt]`` - executes command `cmd` inside + VNF; Optional timeout defines number of seconds to wait until ``prompt`` is detected. + Optional ``prompt`` defines a string, which is used as detection of successful command + execution. In case that prompt is not defined, then content of ``GUEST_PROMPT_LOGIN`` + parameter will be used. Method returns command output as a string. Examples: .. code-block:: python - ['vnf1', 'start'] - ['vnf2', 'start'] - ['vnf2', 'stop'] - ['vnf1', 'stop'] + ['vnf1', 'start'], + ['vnf2', 'start'], + ['vnf1', 'execute_and_wait', 'ifconfig eth0 5.5.5.1/24 up'], + ['vnf2', 'execute_and_wait', 'ifconfig eth0 5.5.5.2/24 up', 120, 'root.*#'], + ['vnf2', 'execute_and_wait', 'ping -c1 5.5.5.1'], + ['vnf2', 'stop'], + ['vnf1', 'stop'], * ``trafficgen`` - triggers traffic generation @@ -119,6 +139,8 @@ of supported objects and their most common functions follows: and given ``traffic`` dictionary. More details about ``traffic`` dictionary and its possible values are available at :ref:`Traffic Generator Integration Guide ` + * ``get_results`` - returns dictionary with results collected from previous execution + of ``send_traffic`` Examples: @@ -126,7 +148,12 @@ of supported objects and their most common functions follows: ['trafficgen', 'send_traffic', {'traffic_type' : 'rfc2544_throughput'}] - ['trafficgen', 'send_traffic', {'traffic_type' : 'rfc2544_back2back', 'bidir' : 'True'}] + ['trafficgen', 'send_traffic', {'traffic_type' : 'rfc2544_back2back', 'bidir' : 'True'}], + ['trafficgen', 'get_results'], + ['tools', 'assert', '#STEP[-1][0]["frame_loss_percent"] < 0.05'], + + +.. _step-driven-tests-variable-usage: * ``settings`` - reads or modifies VSPERF configuration @@ -229,8 +256,7 @@ of supported objects and their most common functions follows: in case that condition is not ``True`` * ``Eval expression`` - evaluates given expression as a python code and returns its result - * ``Exec_Shell command [regex]`` - executes a shell command and filters its output by - (optional) regular expression + * ``Exec_Shell command`` - executes a shell command * ``Exec_Python code`` - executes a python code @@ -260,6 +286,23 @@ of supported objects and their most common functions follows: ['sleep', '60'] + * ``log level message`` - is used to log ``message`` of given ``level`` into vsperf output. + Level is one of info, debug, warning or error. + + Examples: + + .. code-block:: python + + ['log', 'error', 'tools $TOOLS'] + + * ``pdb`` - executes python debugger + + Examples: + + .. code-block:: python + + ['pdb'] + Test Macros ----------- @@ -296,6 +339,15 @@ functionality: ['vswitch', 'del_port', 'int_br0', '#STEP[-1][0]'], # STEP 2 +Another option to refer to previous values, is to define an alias for given step +by its first argument with '#' prefix. Alias must be unique and it can't be a number. +Example of step alias usage: + +.. code-block:: python + + ['#port1', 'vswitch', 'add_vport', 'int_br0'], + ['vswitch', 'del_port', 'int_br0', '#STEP[port1][0]'], + Also commonly used steps can be created as a separate profile. .. code-block:: python @@ -324,6 +376,28 @@ This profile can then be used inside other testcases STEP_VSWITCH_PVP_FINIT } +It is possible to refer to vsperf configuration parameters within step macros. Please +see :ref:`step-driven-tests-variable-usage` for more details. + +In case that step returns a string or list of strings, then it is possible to +filter such output by regular expression. This optional filter can be specified +as a last step parameter with prefix '|'. Output will be split into separate lines +and only matching records will be returned. It is also possible to return a specified +group of characters from the matching lines, e.g. by regex ``|ID (\d+)``. + +Examples: + +.. code-block:: python + + ['tools', 'exec_shell', "sudo $TOOLS['ovs-appctl'] dpif-netdev/pmd-rxq-show", + '|dpdkvhostuser0\s+queue-id: \d'], + ['tools', 'assert', 'len(#STEP[-1])==1'], + + ['vnf', 'execute_and_wait', 'ethtool -L eth0 combined 2'], + ['vnf', 'execute_and_wait', 'ethtool -l eth0', '|Combined:\s+2'], + ['tools', 'assert', 'len(#STEP[-1])==2'] + + HelloWorld and other basic Testcases ------------------------------------ diff --git a/testcases/testcase.py b/testcases/testcase.py index 01b3a975..1b8bf7dc 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -68,7 +68,6 @@ class TestCase(object): self._loadgen = None self._output_file = None self._tc_results = None - self._settings_original = {} self._settings_paths_modified = False self._testcast_run_time = None self._versions = [] @@ -76,21 +75,18 @@ class TestCase(object): self._step_check = False # by default don't check result for step driven testcases self._step_vnf_list = {} self._step_result = [] + self._step_result_mapping = {} self._step_status = None + self._step_send_traffic = False # indication if send_traffic was called within test steps self._testcase_run_time = None - # store all GUEST_ specific settings to keep original values before their expansion - for key in S.__dict__: - if key.startswith('GUEST_'): - self._settings_original[key] = S.getValue(key) - - self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH'))) - self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF'))) - self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN'))) + S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH'))) + S.setValue('VNF', cfg.get('VNF', S.getValue('VNF'))) + S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN'))) test_params = copy.deepcopy(S.getValue('TEST_PARAMS')) tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS')) test_params = merge_spec(test_params, tc_test_params) - self._update_settings('TEST_PARAMS', test_params) + S.setValue('TEST_PARAMS', test_params) S.check_test_params() # override all redefined GUEST_ values to have them expanded correctly @@ -109,6 +105,15 @@ class TestCase(object): self.desc = cfg.get('Description', 'No description given.') self.test = cfg.get('TestSteps', None) + # log testcase name and details + tmp_desc = functions.format_description(self.desc, 50) + self._logger.info('############################################################') + self._logger.info('# Test: %s', self.name) + self._logger.info('# Details: %s', tmp_desc[0]) + for i in range(1, len(tmp_desc)): + self._logger.info('# %s', tmp_desc[i]) + self._logger.info('############################################################') + bidirectional = S.getValue('TRAFFIC')['bidir'] if not isinstance(S.getValue('TRAFFIC')['bidir'], str): raise TypeError( @@ -167,6 +172,7 @@ class TestCase(object): self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2') self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3') self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4') + self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac'] elif len(S.getValue('NICS')) and \ (S.getValue('NICS')[0]['type'] == 'vf' or S.getValue('NICS')[1]['type'] == 'vf'): @@ -186,8 +192,6 @@ class TestCase(object): def run_initialize(self): """ Prepare test execution environment """ - self._logger.debug(self.name) - # mount hugepages if needed self._mount_hugepages() @@ -343,7 +347,7 @@ class TestCase(object): # ...and continue with traffic generation, but keep # in mind, that clean deployment does not configure # OVS nor executes the traffic - if self.deployment != 'clean': + if self.deployment != 'clean' and not self._step_send_traffic: self._traffic_ctl.send_traffic(self._traffic) # dump vswitch flows before they are affected by VNF termination @@ -365,23 +369,6 @@ class TestCase(object): # report test results self.run_report() - # restore original settings - for key in self._settings_original: - S.setValue(key, self._settings_original[key]) - - def _update_settings(self, param, value): - """ Check value of given configuration parameter - In case that new value is different, then testcase - specific settings is updated and original value stored - - :param param: Name of parameter inside settings - :param value: Disired parameter value - """ - orig_value = S.getValue(param) - if orig_value != value: - self._settings_original[param] = copy.deepcopy(orig_value) - S.setValue(param, value) - def _append_results(self, results): """ Method appends mandatory Test Case results to list of dictionaries. @@ -689,18 +676,22 @@ class TestCase(object): if self._step_vnf_list[vnf]: self._step_vnf_list[vnf].stop() - def step_eval_param(self, param, STEP): - # pylint: disable=invalid-name + def step_eval_param(self, param, step_result): """ Helper function for #STEP macro evaluation """ if isinstance(param, str): # evaluate every #STEP reference inside parameter itself - macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param) + macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param) + if macros: for macro in macros: + if macro[1] in self._step_result_mapping: + key = self._step_result_mapping[macro[1]] + else: + key = macro[1] # pylint: disable=eval-used - tmp_val = str(eval(macro[1:])) - param = param.replace(macro, tmp_val) + tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2]))) + param = param.replace(macro[0], tmp_val) # evaluate references to vsperf configuration options macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param) @@ -718,12 +709,12 @@ class TestCase(object): elif isinstance(param, list) or isinstance(param, tuple): tmp_list = [] for item in param: - tmp_list.append(self.step_eval_param(item, STEP)) + tmp_list.append(self.step_eval_param(item, step_result)) return tmp_list elif isinstance(param, dict): tmp_dict = {} for (key, value) in param.items(): - tmp_dict[key] = self.step_eval_param(value, STEP) + tmp_dict[key] = self.step_eval_param(value, step_result) return tmp_dict else: return param @@ -737,6 +728,7 @@ class TestCase(object): eval_params.append(self.step_eval_param(param, step_result)) return eval_params + # pylint: disable=too-many-locals, too-many-branches, too-many-statements def step_run(self): """ Execute actions specified by TestSteps list @@ -758,6 +750,30 @@ class TestCase(object): # run test step by step... for i, step in enumerate(self.test): step_ok = not self._step_check + step_check = self._step_check + regex = None + # configure step result mapping if step alias/label is detected + if step[0].startswith('#'): + key = step[0][1:] + if key.isdigit(): + raise RuntimeError('Step alias can\'t be an integer value {}'.format(key)) + if key in self._step_result_mapping: + raise RuntimeError('Step alias {} has been used already for step ' + '{}'.format(key, self._step_result_mapping[key])) + self._step_result_mapping[step[0][1:]] = i + step = step[1:] + + # store regex filter if it is specified + if isinstance(step[-1], str) and step[-1].startswith('|'): + # evalute macros and variables used in regex + regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0] + step = step[:-1] + + # check if step verification should be suppressed + if step[0].startswith('!'): + step_check = False + step_ok = True + step[0] = step[0][1:] if step[0] == 'vswitch': test_object = self._vswitch_ctl.get_vswitch() elif step[0] == 'namespace': @@ -777,6 +793,9 @@ class TestCase(object): tmp_traffic = copy.deepcopy(self._traffic) tmp_traffic.update(step[2]) step[2] = tmp_traffic + # store indication that traffic has been sent + # so it is not sent again after the execution of teststeps + self._step_send_traffic = True elif step[0].startswith('vnf'): if not self._step_vnf_list[step[0]]: # initialize new VM @@ -790,6 +809,15 @@ class TestCase(object): self._logger.debug("Sleep %s seconds", step[1]) time.sleep(int(step[1])) continue + elif step[0] == 'log': + test_object = self._logger + # there isn't a need for validation of log entry + step_check = False + step_ok = True + elif step[0] == 'pdb': + import pdb + pdb.set_trace() + continue else: self._logger.error("Unsupported test object %s", step[0]) self._step_status = {'status' : False, 'details' : ' '.join(step)} @@ -798,7 +826,7 @@ class TestCase(object): return False test_method = getattr(test_object, step[1]) - if self._step_check: + if step_check: test_method_check = getattr(test_object, CHECK_PREFIX + step[1]) else: test_method_check = None @@ -809,18 +837,24 @@ class TestCase(object): # to support negative indexes step_params = self.step_eval_params(step[2:], self._step_result[:i]) step_log = '{} {}'.format(' '.join(step[:2]), step_params) + step_log += ' filter "{}"'.format(regex) if regex else '' self._logger.debug("Step %s '%s' start", i, step_log) self._step_result[i] = test_method(*step_params) + if regex: + # apply regex to step output + self._step_result[i] = functions.filter_output( + self._step_result[i], regex) + self._logger.debug("Step %s '%s' results '%s'", i, step_log, self._step_result[i]) time.sleep(S.getValue('TEST_STEP_DELAY')) - if self._step_check: + if step_check: step_ok = test_method_check(self._step_result[i], *step_params) except (AssertionError, AttributeError, IndexError) as ex: step_ok = False self._logger.error("Step %s raised %s", i, type(ex).__name__) - if self._step_check: + if step_check: self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok) if not step_ok: diff --git a/tools/functions.py b/tools/functions.py index e8bc31da..9292867d 100644 --- a/tools/functions.py +++ b/tools/functions.py @@ -19,6 +19,7 @@ import os import logging import glob import shutil +import re from conf import settings as S MAX_L4_FLOWS = 65536 @@ -171,3 +172,44 @@ def check_traffic(traffic): traffic['multistream'] = MAX_L4_FLOWS return traffic + +def filter_output(output, regex): + """Filter output by defined regex. Output can be either string, list or tuple. + Every string is split into list line by line. After that regex is applied + to filter only matching lines, which are returned back. + + :returns: list of matching records + """ + result = [] + if isinstance(output, str): + for line in output.split('\n'): + result += re.findall(regex, line) + return result + elif isinstance(output, list) or isinstance(output, tuple): + tmp_res = [] + for item in output: + tmp_res.append(filter_output(item, regex)) + return tmp_res + else: + raise RuntimeError('Only strings and lists are supported by filter_output(), ' + 'but output has type {}'.format(type(output))) + +def format_description(desc, length): + """ Split description into multiple lines based on given line length. + + :param desc: A string with testcase description + :param length: A maximum line length + """ + # split description to multiple lines + words = desc.split() + output = [] + line = '' + for word in words: + if len(line) + len(word) < length: + line += '{} '.format(word) + else: + output.append(line.strip()) + line = '{} '.format(word) + + output.append(line.strip()) + return output diff --git a/tools/pkt_gen/ixnet/ixnet.py b/tools/pkt_gen/ixnet/ixnet.py index 972fa331..b8fb1879 100755 --- a/tools/pkt_gen/ixnet/ixnet.py +++ b/tools/pkt_gen/ixnet/ixnet.py @@ -153,9 +153,8 @@ class IxNet(trafficgen.ITrafficGenerator): """Initialize IXNET members """ super().__init__() - self._script = os.path.join(settings.getValue('TRAFFICGEN_IXIA_3RD_PARTY'), - settings.getValue('TRAFFICGEN_IXNET_TCL_SCRIPT')) self._tclsh = tkinter.Tcl() + self._script = None self._cfg = None self._logger = logging.getLogger(__name__) self._params = None @@ -177,6 +176,8 @@ class IxNet(trafficgen.ITrafficGenerator): def configure(self): """Configure system for IxNetwork. """ + self._script = os.path.join(settings.getValue('TRAFFICGEN_IXIA_3RD_PARTY'), + settings.getValue('TRAFFICGEN_IXNET_TCL_SCRIPT')) self._cfg = { 'lib_path': settings.getValue('TRAFFICGEN_IXNET_LIB_PATH'), # IxNetwork machine configuration @@ -225,6 +226,8 @@ class IxNet(trafficgen.ITrafficGenerator): 'multipleStreams': traffic['multistream'], 'streamType': traffic['stream_type'], 'rfc2544TestType': 'throughput', + 'flowControl': "True" if traffic['flow_control'] else "False", + 'learningFrames': "True" if traffic['learning_frames'] else "False", } self._params['traffic'] = self.traffic_defaults.copy() @@ -280,6 +283,8 @@ class IxNet(trafficgen.ITrafficGenerator): 'multipleStreams': traffic['multistream'], 'streamType': traffic['stream_type'], 'rfc2544TestType': 'throughput', + 'flowControl': "True" if traffic['flow_control'] else "False", + 'learningFrames': "True" if traffic['learning_frames'] else "False", } self._params['traffic'] = self.traffic_defaults.copy() @@ -418,6 +423,8 @@ class IxNet(trafficgen.ITrafficGenerator): 'multipleStreams': traffic['multistream'], 'streamType': traffic['stream_type'], 'rfc2544TestType': 'back2back', + 'flowControl': "True" if traffic['flow_control'] else "False", + 'learningFrames': "True" if traffic['learning_frames'] else "False", } self._params['traffic'] = self.traffic_defaults.copy() diff --git a/tools/teststepstools.py b/tools/teststepstools.py index 5d551c68..639e3437 100644 --- a/tools/teststepstools.py +++ b/tools/teststepstools.py @@ -15,10 +15,10 @@ """Various helper functions for step driven testcases """ -import re import logging import subprocess import locale +from tools.functions import filter_output _LOGGER = logging.getLogger(__name__) @@ -93,11 +93,7 @@ class TestStepsTools(object): output = output.decode(locale.getdefaultlocale()[1]) if regex: - for line in output.split('\n'): - result = re.findall(regex, line) - if result: - return result - return [] + return filter_output(output, regex) return output diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py index 30f073ee..8e3d44de 100644 --- a/vnfs/qemu/qemu.py +++ b/vnfs/qemu/qemu.py @@ -146,13 +146,14 @@ class IVnfQemu(IVnf): """ if self.is_running(): try: - # exit testpmd if needed - if self._guest_loopback == 'testpmd': - self.execute_and_wait('stop', 120, "Done") - self.execute_and_wait('quit', 120, "[bB]ye") + if self._login_active: + # exit testpmd if needed + if self._guest_loopback == 'testpmd': + self.execute_and_wait('stop', 120, "Done") + self.execute_and_wait('quit', 120, "[bB]ye") - # turn off VM - self.execute_and_wait('poweroff', 120, "Power down") + # turn off VM + self.execute_and_wait('poweroff', 120, "Power down") except pexpect.TIMEOUT: self.kill() @@ -169,7 +170,7 @@ class IVnfQemu(IVnf): # helper functions - def _login(self, timeout=120): + def login(self, timeout=120): """ Login to QEMU instance. @@ -178,8 +179,11 @@ class IVnfQemu(IVnf): :param timeout: Timeout to wait for login to complete. - :returns: None + :returns: True if login is active """ + if self._login_active: + return self._login_active + # if no timeout was set, we likely started QEMU without waiting for it # to boot. This being the case, we best check that it has finished # first. @@ -191,21 +195,8 @@ class IVnfQemu(IVnf): self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number]) self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5) - - def send_and_pass(self, cmd, timeout=30): - """ - Send ``cmd`` and wait ``timeout`` seconds for it to pass. - - :param cmd: Command to send to guest. - :param timeout: Time to wait for prompt before checking return code. - - :returns: None - """ - self.execute(cmd) - self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout) - self.execute('echo $?') - self._child.expect('^0$', timeout=1) # expect a 0 - self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout) + self._login_active = True + return self._login_active def _affinitize(self): """ @@ -277,29 +268,31 @@ class IVnfQemu(IVnf): """ Configure VM to run VNF, e.g. port forwarding application based on the configuration """ + if self._guest_loopback == 'buildin': + return + + self.login() + if self._guest_loopback == 'testpmd': - self._login() self._configure_testpmd() elif self._guest_loopback == 'l2fwd': - self._login() self._configure_l2fwd() elif self._guest_loopback == 'linux_bridge': - self._login() self._configure_linux_bridge() - elif self._guest_loopback != 'buildin': - self._logger.error('Unsupported guest loopback method "%s" was specified. Option' - ' "buildin" will be used as a fallback.', self._guest_loopback) + elif self._guest_loopback != 'clean': + raise RuntimeError('Unsupported guest loopback method "%s" was specified.', + self._guest_loopback) def wait(self, prompt=None, timeout=30): if prompt is None: prompt = S.getValue('GUEST_PROMPT')[self._number] - super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout) + return super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout) def execute_and_wait(self, cmd, timeout=30, prompt=None): if prompt is None: prompt = S.getValue('GUEST_PROMPT')[self._number] - super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout, - prompt=prompt) + return super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout, + prompt=prompt) def _modify_dpdk_makefile(self): """ @@ -393,7 +386,7 @@ class IVnfQemu(IVnf): 'VSWITCH_JUMBO_FRAMES_SIZE')) self.execute_and_wait('./testpmd {}'.format(testpmd_params), 60, "Done") - self.execute('set fwd ' + self._testpmd_fwd_mode, 1) + self.execute_and_wait('set fwd ' + self._testpmd_fwd_mode, 20, 'testpmd>') self.execute_and_wait('start', 20, 'testpmd>') def _configure_l2fwd(self): @@ -407,12 +400,12 @@ class IVnfQemu(IVnf): # configure all interfaces for nic in self._nics: - self.execute('ip addr add ' + - nic['ip'] + ' dev ' + nic['device']) + self.execute_and_wait('ip addr add ' + + nic['ip'] + ' dev ' + nic['device']) if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'): - self.execute('ifconfig {} mtu {}'.format( + self.execute_and_wait('ifconfig {} mtu {}'.format( nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE'))) - self.execute('ip link set dev ' + nic['device'] + ' up') + self.execute_and_wait('ip link set dev ' + nic['device'] + ' up') # build and configure system for l2fwd self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + @@ -437,43 +430,42 @@ class IVnfQemu(IVnf): self._configure_disable_firewall() # configure linux bridge - self.execute('brctl addbr br0') + self.execute_and_wait('brctl addbr br0') # add all NICs into the bridge for nic in self._nics: - self.execute('ip addr add ' + - nic['ip'] + ' dev ' + nic['device']) + self.execute_and_wait('ip addr add ' + nic['ip'] + ' dev ' + nic['device']) if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'): - self.execute('ifconfig {} mtu {}'.format( + self.execute_and_wait('ifconfig {} mtu {}'.format( nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE'))) - self.execute('ip link set dev ' + nic['device'] + ' up') - self.execute('brctl addif br0 ' + nic['device']) + self.execute_and_wait('ip link set dev ' + nic['device'] + ' up') + self.execute_and_wait('brctl addif br0 ' + nic['device']) - self.execute('ip addr add ' + - S.getValue('GUEST_BRIDGE_IP')[self._number] + - ' dev br0') - self.execute('ip link set dev br0 up') + self.execute_and_wait('ip addr add {} dev br0'.format( + S.getValue('GUEST_BRIDGE_IP')[self._number])) + self.execute_and_wait('ip link set dev br0 up') # Add the arp entries for the IXIA ports and the bridge you are using. # Use command line values if provided. trafficgen_mac = S.getValue('VANILLA_TGEN_PORT1_MAC') trafficgen_ip = S.getValue('VANILLA_TGEN_PORT1_IP') - self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) + self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) trafficgen_mac = S.getValue('VANILLA_TGEN_PORT2_MAC') trafficgen_ip = S.getValue('VANILLA_TGEN_PORT2_IP') - self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) + self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) # Enable forwarding - self.execute('sysctl -w net.ipv4.ip_forward=1') + self.execute_and_wait('sysctl -w net.ipv4.ip_forward=1') # Controls source route verification # 0 means no source validation - self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0') + self.execute_and_wait('sysctl -w net.ipv4.conf.all.rp_filter=0') for nic in self._nics: - self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0') + self.execute_and_wait('sysctl -w net.ipv4.conf.' + nic['device'] + + '.rp_filter=0') def _bind_dpdk_driver(self, driver, pci_slots): """ diff --git a/vnfs/qemu/qemu_dpdk_vhost_user.py b/vnfs/qemu/qemu_dpdk_vhost_user.py index 93147838..b53af6fa 100644 --- a/vnfs/qemu/qemu_dpdk_vhost_user.py +++ b/vnfs/qemu/qemu_dpdk_vhost_user.py @@ -68,11 +68,14 @@ class QemuDpdkVhostUser(IVnfQemu): else: vhost_folder = S.getValue('TOOLS')['ovs_var_tmp'] - nic_mode = '' if S.getValue('VSWITCH_VHOSTUSER_SERVER_MODE') else ',server' + if S.getValue('VSWITCH_VHOSTUSER_SERVER_MODE'): + nic_name = 'dpdkvhostuser' + ifi + else: + nic_name = 'dpdkvhostuserclient' + ifi + ',server' + self._cmd += ['-chardev', 'socket,id=char' + ifi + - ',path=' + vhost_folder + - 'dpdkvhostuser' + ifi + nic_mode, + ',path=' + vhost_folder + nic_name, '-netdev', 'type=vhost-user,id=' + net + ',chardev=char' + ifi + ',vhostforce' + queue_str, diff --git a/vnfs/vnf/vnf.py b/vnfs/vnf/vnf.py index 759bdd66..5ac2ada3 100644 --- a/vnfs/vnf/vnf.py +++ b/vnfs/vnf/vnf.py @@ -17,6 +17,7 @@ Interface for VNF. """ import time +import pexpect from tools import tasks class IVnf(tasks.Process): @@ -40,14 +41,7 @@ class IVnf(tasks.Process): self._number + 1, self._number) IVnf._number_vnfs = IVnf._number_vnfs + 1 self._log_prefix = 'vnf_%d_cmd : ' % self._number - - def start(self): - """ - Starts VNF instance. - - This is a blocking function - """ - super(IVnf, self).start() + self._login_active = False def stop(self): """ @@ -60,6 +54,18 @@ class IVnf(tasks.Process): # sporadic reboot of host. (caused by hugepages or DPDK ports) super(IVnf, self).kill(signal='-9', sleep=10) + def login(self, dummy_timeout=120): + """ + Login to GUEST instance. + + This can be used after booting the machine + + :param timeout: Timeout to wait for login to complete. + + :returns: True if login is active + """ + raise NotImplementedError() + def execute(self, cmd, delay=0): """ execute ``cmd`` with given ``delay``. @@ -77,6 +83,14 @@ class IVnf(tasks.Process): :returns: None. """ self._logger.debug('%s%s', self._log_prefix, cmd) + + # ensure that output from previous commands is flushed + try: + while not self._child.expect(r'.+', timeout=0.1): + pass + except (pexpect.TIMEOUT, pexpect.EOF): + pass + self._child.sendline(cmd) time.sleep(delay) @@ -95,10 +109,10 @@ class IVnf(tasks.Process): Please note that this value can be floating point which allows to pass milliseconds. - :returns: True if result_cmd has been detected before - timeout has been reached, False otherwise. + :returns: output of executed command """ self._child.expect(prompt, timeout=timeout) + return self._child.before def execute_and_wait(self, cmd, timeout=30, prompt=''): """ @@ -119,26 +133,36 @@ class IVnf(tasks.Process): ``prompt`` passed in __init__ method will be used. - :returns: True if end of execution has been detected - before timeout has been reached, False otherwise. + :returns: output of executed command """ self.execute(cmd) - self.wait(prompt=prompt, timeout=timeout) + return self.wait(prompt=prompt, timeout=timeout) - # pylint: disable=simplifiable-if-statement - def validate_start(self, dummy_result): + def validate_start(self, dummyresult): """ Validate call of VNF start() """ if self._child and self._child.isalive(): return True - else: - return False + + return False def validate_stop(self, result): - """ Validate call of fVNF stop() + """ Validate call of VNF stop() """ return not self.validate_start(result) + @staticmethod + def validate_execute_and_wait(result, dummy_cmd, dummy_timeout=30, dummy_prompt=''): + """ Validate command execution within VNF + """ + return len(result) > 0 + + @staticmethod + def validate_login(result): + """ Validate successful login into guest + """ + return result + @staticmethod def reset_vnf_counter(): """ diff --git a/vsperf b/vsperf index 6efe53da..8125ba43 100755 --- a/vsperf +++ b/vsperf @@ -444,18 +444,36 @@ def handle_list_options(args): sys.exit(0) if args['list']: - # configure tests - if args['integration']: - testcases = settings.getValue('INTEGRATION_TESTS') + list_testcases(args) + sys.exit(0) + + +def list_testcases(args): + """ Print list of testcases requested by --list CLI argument + + :param args: A dictionary with all CLI arguments + """ + # configure tests + if args['integration']: + testcases = settings.getValue('INTEGRATION_TESTS') + else: + testcases = settings.getValue('PERFORMANCE_TESTS') + + print("Available Tests:") + print("================") + + for test in testcases: + description = functions.format_description(test['Description'], 70) + if len(test['Name']) < 40: + print('* {:40} {}'.format('{}:'.format(test['Name']), description[0])) else: - testcases = settings.getValue('PERFORMANCE_TESTS') + print('* {}'.format('{}:'.format(test['Name']))) + print(' {:40} {}'.format('', description[0])) + for i in range(1, len(description)): + print(' {:40} {}'.format('', description[i])) + - print("Available Tests:") - print("================") - for test in testcases: - print('* %-30s %s' % ('%s:' % test['Name'], test['Description'])) - sys.exit(0) def vsperf_finalize(): @@ -678,6 +696,7 @@ def main(): # testcases.integration.IntegrationTestCase to testcases.performance.PerformanceTestCase # pylint: disable=redefined-variable-type suite = unittest.TestSuite() + settings_snapshot = copy.deepcopy(settings.__dict__) for cfg in selected_tests: test_name = cfg.get('Name', '') try: @@ -692,6 +711,8 @@ def main(): _LOGGER.exception("Failed to run test: %s", test_name) suite.addTest(MockTestCase(str(ex), False, test_name)) _LOGGER.info("Continuing with next test...") + finally: + settings.restore_from_dict(settings_snapshot) # generate final rst report with results of all executed TCs generate_final_report() diff --git a/vswitches/ovs.py b/vswitches/ovs.py index 7e16c142..76cabb0d 100644 --- a/vswitches/ovs.py +++ b/vswitches/ovs.py @@ -91,6 +91,27 @@ class IVSwitchOvs(IVSwitch, tasks.Process): self._logger.info("Vswitchd...Started.") + def restart(self): + """ Restart ``ovs-vswitchd`` instance. ``ovsdb-server`` is not restarted. + + :raises: pexpect.EOF, pexpect.TIMEOUT + """ + self._logger.info("Restarting vswitchd...") + if os.path.isfile(self._vswitchd_pidfile_path): + self._logger.info('Killing ovs-vswitchd...') + with open(self._vswitchd_pidfile_path, "r") as pidfile: + vswitchd_pid = pidfile.read().strip() + tasks.terminate_task(vswitchd_pid, logger=self._logger) + + try: + tasks.Process.start(self) + self.relinquish() + except (pexpect.EOF, pexpect.TIMEOUT) as exc: + logging.error("Exception during VSwitch start.") + self._kill_ovsdb() + raise exc + self._logger.info("Vswitchd...Started.") + def configure(self): """ Configure vswitchd through ovsdb if needed """ @@ -109,6 +130,7 @@ class IVSwitchOvs(IVSwitch, tasks.Process): """ self._logger.info("Terminating vswitchd...") self.kill() + self._bridges = {} self._logger.info("Vswitchd...Terminated.") def add_switch(self, switch_name, params=None): @@ -488,3 +510,8 @@ class IVSwitchOvs(IVSwitch, tasks.Process): """ bridge = self._bridges[switch_name] return 'stp_enable : true' in ''.join(bridge.bridge_info()) + + def validate_restart(self, dummy_result): + """ Validate restart + """ + return True diff --git a/vswitches/ovs_dpdk_vhost.py b/vswitches/ovs_dpdk_vhost.py index 3b20be35..11b32c88 100644 --- a/vswitches/ovs_dpdk_vhost.py +++ b/vswitches/ovs_dpdk_vhost.py @@ -149,7 +149,7 @@ class OvsDpdkVhost(IVSwitchOvs): nic_type = 'dpdkvhostuserclient' vhost_count = self._get_port_count('type={}'.format(nic_type)) - port_name = 'dpdkvhostuser' + str(vhost_count) + port_name = nic_type + str(vhost_count) params = ['--', 'set', 'Interface', port_name, 'type={}'.format(nic_type)] if not S.getValue('VSWITCH_VHOSTUSER_SERVER_MODE'): params += ['--', 'set', 'Interface', port_name, 'options:vhost-server-path=' diff --git a/vswitches/vswitch.py b/vswitches/vswitch.py index dd69e6d9..efa3a349 100644 --- a/vswitches/vswitch.py +++ b/vswitches/vswitch.py @@ -36,6 +36,13 @@ class IVSwitch(object): """ raise NotImplementedError() + def restart(self): + """Retart the vSwitch + + Restart of vSwitch is required for failover testcases. + """ + raise NotImplementedError() + def stop(self): """Stop the vSwitch -- cgit 1.2.3-korg