diff options
| -rwxr-xr-x | ci/build-vsperf.sh | 2 | ||||
| -rw-r--r-- | conf/03_traffic.conf | 5 | ||||
| -rw-r--r-- | conf/10_custom.conf | 4 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst | 108 | ||||
| -rw-r--r-- | docs/testing/user/configguide/trafficgen.rst | 25 | ||||
| -rw-r--r-- | tools/pkt_gen/trex/trex.py | 53 |
6 files changed, 173 insertions, 24 deletions
diff --git a/ci/build-vsperf.sh b/ci/build-vsperf.sh index 00a548ba..a8da6a80 100755 --- a/ci/build-vsperf.sh +++ b/ci/build-vsperf.sh @@ -535,7 +535,7 @@ case $1 in echo "VSPERF merge job" echo "================" - execute_pylint_check + execute_vsperf_pylint_check terminate_vsperf execute_vsperf_sanity terminate_vsperf diff --git a/conf/03_traffic.conf b/conf/03_traffic.conf index 67318893..8aff2e35 100644 --- a/conf/03_traffic.conf +++ b/conf/03_traffic.conf @@ -487,6 +487,11 @@ TRAFFICGEN_TREX_LEARNING_MODE = True TRAFFICGEN_TREX_LEARNING_DURATION = 5 # FOR SR-IOV or multistream layer 2 tests to work with T-Rex enable Promiscuous mode TRAFFICGEN_TREX_PROMISCUOUS = False +# Enable below options to force T-rex api to attempt to use speed specified on server +# side when pushing traffic. For 40G use 40000. For 25G use 25000. +TRAFFICGEN_TREX_FORCE_PORT_SPEED = False +TRAFFICGEN_TREX_PORT_SPEED = 10000 # 10G + PATHS['trafficgen'] = { 'Trex': { 'type' : 'src', diff --git a/conf/10_custom.conf b/conf/10_custom.conf index 917d16b4..0e274aab 100644 --- a/conf/10_custom.conf +++ b/conf/10_custom.conf @@ -138,6 +138,10 @@ TRAFFICGEN_TREX_LEARNING_MODE = True TRAFFICGEN_TREX_LEARNING_DURATION = 5 # FOR SR-IOV or multistream layer 2 tests to work with T-Rex enable Promiscuous mode TRAFFICGEN_TREX_PROMISCUOUS = False +# Enable below options to force T-rex api to attempt to use speed specified on server +# side when pushing traffic. For 40G use 40000. For 25G use 25000. +TRAFFICGEN_TREX_FORCE_PORT_SPEED = False +TRAFFICGEN_TREX_PORT_SPEED = 10000 # 10G # TRex validation option for RFC2544 TRAFFICGEN_TREX_VERIFICATION_MODE = False TRAFFICGEN_TREX_VERIFICATION_DURATION = 60 diff --git a/docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst b/docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst index e1372520..c703ff40 100644 --- a/docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst +++ b/docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst @@ -413,7 +413,21 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoak **Title**: RFC 2889 X% packet loss Max Forwarding Rate Soak Test - **Prerequisite Test** LTD.Throughput.RFC2544.PacketLossRatio + **Prerequisite Tests**: + + LTD.Throughput.RFC2544.PacketLossRatio will determine the offered load and + frame size for which the maximum theoretical throughput of the interface + has not been achieved. As described in RFC 2544 section 24, the final + determination of the benchmark SHOULD be conducted using a full length + trial, and for this purpose the duration is 5 minutes with zero loss ratio. + + It is also essential to verify that the Traffic Generator has sufficient + stability to conduct Soak tests. Therefore, a prerequisite is to perform + this test with the DUT removed and replaced with a cross-over cable (or + other equivalent very low overhead method such as a loopback in a HW switch), + so that the traffic generator (and any other network involved) can be tested + over the Soak period. Note that this test may be challenging for software- + based traffic generators. **Priority**: @@ -422,12 +436,19 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoak The aim of this test is to understand the Max Forwarding Rate stability over an extended test duration in order to uncover any outliers. To allow for an extended test duration, the test should ideally run for 24 hours - or, if this is not possible, for at least 6 hours. For this test, each frame - size must be sent at the highest Throughput rate with X% packet loss, as - determined in the prerequisite test. The default loss percentages to be - tested are: - X = 0% - X = 10^-7% + or if this is not possible, for at least 6 hours. - Note: Other values can be tested if required by the user. + For this test, one frame size must be sent at the highest frame rate with + X% packet loss ratio, as determined in the prerequisite test (a short trial). + The loss ratio shall be measured and recorded every 5 minutes during the test + (it may be sufficient to collect lost frame counts and divide by the number + of frames sent in 5 minutes to see if a threshold has been crossed, + and accept some small inaccuracy in the threshold evaluation, not the result). + The default loss ratio is X = 0% and loss ratio > 10^-7% is the default + threshold to terminate the test early (or inform the test operator of + the failure status). + + Note: Other values of X and loss threshold can be tested if required by the user. **Expected Result**: @@ -441,13 +462,13 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoak and reporting any time intervals with packet loss. The `RFC2889 <https://www.rfc-editor.org/rfc/rfc2289.txt>`__ Forwarding Rate shall be measured in each interval. - An interval of 60s is suggested. + An interval of 300s is suggested. - CPU and memory utilization may also be collected as part of this test, to determine the vSwitch's performance footprint on the system. - The `RFC5481 <https://www.rfc-editor.org/rfc/rfc5481.txt>`__ PDV form of delay variation on the traffic flow, - using the 99th percentile. + using the 99th percentile, may also be collected. .. 3.2.2.1.7 @@ -457,7 +478,22 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoakFrameModification **Title**: RFC 2889 Max Forwarding Rate Soak Test with Frame Modification **Prerequisite Test**: + LTD.Throughput.RFC2544.PacketLossRatioFrameModification (0% Packet Loss) + will determine the offered load and + frame size for which the maximum theoretical throughput of the interface + has not been achieved. As described in RFC 2544 section 24, the final + determination of the benchmark SHOULD be conducted using a full length + trial, and for this purpose the duration is 5 minutes with zero loss ratio. + + It is also essential to verify that the Traffic Generator has sufficient + stability to conduct Soak tests. Therefore, a prerequisite is to perform + this test with the DUT removed and replaced with a cross-over cable (or + other equivalent very low overhead method such as a loopback in a HW switch), + so that the traffic generator (and any other network involved) can be tested + over the Soak period. Note that this test may be challenging for software- + based traffic generators. + **Priority**: @@ -466,9 +502,19 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoakFrameModification The aim of this test is to understand the Max Forwarding Rate stability over an extended test duration in order to uncover any outliers. To allow for an extended test duration, the test should ideally run for 24 hours or, if - this is not possible, for at least 6 hour. For this test, each frame - size must be sent at the highest Throughput rate with 0% packet loss, as - determined in the prerequisite test. + this is not possible, for at least 6 hours. + + For this test, one frame size must be sent at the highest frame rate with + X% packet loss ratio, as determined in the prerequisite test (a short trial). + The loss ratio shall be measured and recorded every 5 minutes during the test + (it may be sufficient to collect lost frame counts and divide by the number + of frames sent in 5 minutes to see if a threshold has been crossed, + and accept some small inaccuracy in the threshold evaluation, not the result). + The default loss ratio is X = 0% and loss ratio > 10^-7% is the default + threshold to terminate the test early (or inform the test operator of + the failure status). + + Note: Other values of X and loss threshold can be tested if required by the user. During this test, the DUT must perform the following operations on the traffic flow: @@ -498,13 +544,13 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoakFrameModification and reporting any time intervals with packet loss. The `RFC2889 <https://www.rfc-editor.org/rfc/rfc2289.txt>`__ Forwarding Rate shall be measured in each interval. - An interval of 60s is suggested. + An interval of 300s is suggested. - CPU and memory utilization may also be collected as part of this test, to determine the vSwitch's performance footprint on the system. - The `RFC5481 <https://www.rfc-editor.org/rfc/rfc5481.txt>`__ PDV form of delay variation on the traffic flow, using the 99th - percentile. + percentile, may also be collected. .. 3.2.2.1.8 @@ -1150,7 +1196,22 @@ Test ID: LTD.PacketDelayVariation.RFC3393.Soak **Title**: Packet Delay Variation Soak Test - **Prerequisite Tests**: LTD.Throughput.RFC2544.PacketLossRatio (0% Packet Loss) + **Prerequisite Tests**: + + LTD.Throughput.RFC2544.PacketLossRatio will determine the offered load and + frame size for which the maximum theoretical throughput of the interface + has not been achieved. As described in RFC 2544 section 24, the final + determination of the benchmark SHOULD be conducted using a full length + trial, and for this purpose the duration is 5 minutes with zero loss ratio. + + It is also essential to verify that the Traffic Generator has sufficient + stability to conduct Soak tests. Therefore, a prerequisite is to perform + this test with the DUT removed and replaced with a cross-over cable (or + other equivalent very low overhead method such as a loopback in a HW switch), + so that the traffic generator (and any other network involved) can be tested + over the Soak period. Note that this test may be challenging for software- + based traffic generators. + **Priority**: @@ -1160,9 +1221,20 @@ Test ID: LTD.PacketDelayVariation.RFC3393.Soak variation for different frame sizes over an extended test duration and to determine if there are any outliers. To allow for an extended test duration, the test should ideally run for 24 hours or, if this is not - possible, for at least 6 hour. For this test, each frame size must be - sent at the highest possible throughput with 0% packet loss, as - determined in the prerequisite test. + possible, for at least 6 hours. + + For this test, one frame size must be sent at the highest frame rate with + X% packet loss ratio, as determined in the prerequisite test (a short trial). + The loss ratio shall be measured and recorded every 5 minutes during the test + (it may be sufficient to collect lost frame counts and divide by the number + of frames sent in 5 minutes to see if a threshold has been crossed, + and accept some small inaccuracy in the threshold evaluation, not the result). + The default loss ratio is X = 0% and loss ratio > 10^-7% is the default + threshold to terminate the test early (or inform the test operator of + the failure status). + + Note: Other values of X and loss threshold can be tested if required by the user. + **Expected Result**: @@ -1173,7 +1245,7 @@ Test ID: LTD.PacketDelayVariation.RFC3393.Soak - The packet delay variation value for traffic passing through the DUT. - The `RFC5481 <https://www.rfc-editor.org/rfc/rfc5481.txt>`__ PDV form of delay variation on the traffic flow, - using the 99th percentile, for each 60s interval during the test. + using the 99th percentile, for each 300s interval during the test. - CPU and memory utilization may also be collected as part of this test, to determine the vSwitch's performance footprint on the system. diff --git a/docs/testing/user/configguide/trafficgen.rst b/docs/testing/user/configguide/trafficgen.rst index 33824486..4909c55a 100644 --- a/docs/testing/user/configguide/trafficgen.rst +++ b/docs/testing/user/configguide/trafficgen.rst @@ -577,7 +577,7 @@ http://www.mono-project.com/docs/getting-started/install/linux/ rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF" yum-config-manager --add-repo http://download.mono-project.com/repo/centos/ - yum -y install mono-complete + yum -y install mono-complete-5.8.0.127-0.xamarin.3.epel7.x86_64 To prevent gpg errors on future yum installation of packages the mono-project repo should be disabled once installed. @@ -795,6 +795,13 @@ It is neccesary for proper connection between Trex server and VSPERF. Firewall must allow a connection from DUT (VSPERF) to the T-Rex server running at TCP port 4501. +**NOTE:** For high speed cards it may be advantageous to start T-Rex with more transmit queues/cores. + +.. code-block:: console + + cd trex-cores/scripts/ + ./t-rex-64 -i -c 10 + For additional information about Trex stateless mode see Trex stateless documentation: https://trex-tgn.cisco.com/trex/doc/trex_stateless.html @@ -862,6 +869,22 @@ modified. Enable Promiscuous mode when doing multistream at layer 2 testing with TRAFFICGEN_TREX_PROMISCUOUS=True +Card Bandwidth Options +~~~~~~~~~~~~~~~~~~~~~~ + +T-Rex API will attempt to retrieve the highest possible speed from the card using internal +calls to port information. If you are using two separate cards then it will take the lowest +of the two cards as the max speed. If necessary you can try to force the API to use a +specific maximum speed per port. The below configurations can be adjusted to enable this. + +.. code-block:: console + + TRAFFICGEN_TREX_FORCE_PORT_SPEED = True + TRAFFICGEN_TREX_PORT_SPEED = 40000 # 40 gig + +**Note::** Setting higher than possible speeds will result in unpredictable behavior when running +tests such as duration inaccuracy and/or complete test failure. + RFC2544 Validation ~~~~~~~~~~~~~~~~~~ diff --git a/tools/pkt_gen/trex/trex.py b/tools/pkt_gen/trex/trex.py index cfe54b78..e0ce4c48 100644 --- a/tools/pkt_gen/trex/trex.py +++ b/tools/pkt_gen/trex/trex.py @@ -33,6 +33,7 @@ try: # pylint: disable=wrong-import-position, import-error sys.path.append(settings.getValue('PATHS')['trafficgen']['Trex']['src']['path']) from trex_stl_lib.api import * + from trex_stl_lib import trex_stl_exceptions except ImportError: # VSPERF performs detection of T-Rex api during testcase initialization. So if # T-Rex is requsted and API is not available it will fail before this code @@ -68,6 +69,7 @@ _EMPTY_STATS = { 'tx_pps': 0.0, 'tx_util': 0.0,}} + class Trex(ITrafficGenerator): """Trex Traffic generator wrapper.""" _logger = logging.getLogger(__name__) @@ -84,6 +86,20 @@ class Trex(ITrafficGenerator): self._trex_user = settings.getValue('TRAFFICGEN_TREX_USER') self._stlclient = None self._verification_params = None + self._show_packet_data = False + + def show_packet_info(self, packet_a, packet_b): + """ + Log packet layers to screen + :param packet_a: Scapy.layers packet + :param packet_b: Scapy.layers packet + :return: None + """ + # we only want to show packet data once per test + if self._show_packet_data: + self._show_packet_data = False + self._logger.info(packet_a.show()) + self._logger.info(packet_b.show()) def connect(self): '''Connect to Trex traffic generator @@ -262,11 +278,29 @@ class Trex(ITrafficGenerator): self._stlclient.set_service_mode(ports=my_ports, enabled=False) ports_info = self._stlclient.get_port_info(my_ports) + + # get max support speed + max_speed = 0 + if settings.getValue('TRAFFICGEN_TREX_FORCE_PORT_SPEED'): + max_speed = settings.getValue('TRAFFICGEN_TREX_PORT_SPEED') + elif ports_info[0]['supp_speeds']: + max_speed_1 = max(ports_info[0]['supp_speeds']) + max_speed_2 = max(ports_info[1]['supp_speeds']) + else: + # if max supported speed not in port info or set manually, just assume 10G + max_speed = 10000 + if not max_speed: + # since we can only control both ports at once take the lower of the two + max_speed = min(max_speed_1, max_speed_2) + gbps_speed = (max_speed / 1000) * (float(traffic['frame_rate']) / 100.0) + self._logger.debug('Starting traffic at %s Gpbs speed', gbps_speed) + # for SR-IOV if settings.getValue('TRAFFICGEN_TREX_PROMISCUOUS'): self._stlclient.set_port_attr(my_ports, promiscuous=True) packet_1, packet_2 = Trex.create_packets(traffic, ports_info) + self.show_packet_info(packet_1, packet_2) stream_1, stream_2, stream_1_lat, stream_2_lat = Trex.create_streams(packet_1, packet_2, traffic) self._stlclient.add_streams(stream_1, ports=[0]) self._stlclient.add_streams(stream_2, ports=[1]) @@ -289,7 +323,13 @@ class Trex(ITrafficGenerator): pcap_id[pcap_dir] = self._stlclient.start_capture(**capture) self._stlclient.clear_stats() - self._stlclient.start(ports=my_ports, force=True, duration=duration) + # if the user did not start up T-Rex server with more than default cores, use default mask. + # Otherwise use mask to take advantage of multiple cores. + try: + self._stlclient.start(ports=my_ports, force=True, duration=duration, mult="{}gbps".format(gbps_speed), + core_mask=self._stlclient.CORE_MASK_PIN) + except STLError: + self._stlclient.start(ports=my_ports, force=True, duration=duration, mult="{}gbps".format(gbps_speed)) self._stlclient.wait_on_traffic(ports=my_ports) stats = self._stlclient.get_stats(sync_now=True) @@ -414,9 +454,11 @@ class Trex(ITrafficGenerator): if test_lossrate <= lossrate: # save the last passing trial for verification self._verification_params = copy.deepcopy(new_params) - self._logger.debug("Iteration: %s, frame rate: %s, throughput_rx_fps: %s, frame_loss_percent: %s", - iteration, "{:.3f}".format(new_params['frame_rate']), stats['total']['rx_pps'], - "{:.3f}".format(test_lossrate)) + packets_lost = stats['total']['opackets'] - stats['total']['ipackets'] + self._logger.debug("Iteration: %s, frame rate: %s, throughput_rx_fps: %s," + + " frames lost %s, frame_loss_percent: %s", iteration, + "{:.3f}".format(new_params['frame_rate']), stats['total']['rx_pps'], + packets_lost, "{:.3f}".format(test_lossrate)) if test_lossrate == 0.0 and new_params['frame_rate'] == traffic['frame_rate']: return copy.deepcopy(stats) elif test_lossrate > lossrate: @@ -439,6 +481,8 @@ class Trex(ITrafficGenerator): self._logger.info("In Trex send_cont_traffic method") self._params.clear() + self._show_packet_data = True + self._params['traffic'] = self.traffic_defaults.copy() if traffic: self._params['traffic'] = merge_spec( @@ -467,6 +511,7 @@ class Trex(ITrafficGenerator): """ self._logger.info("In Trex send_rfc2544_throughput method") self._params.clear() + self._show_packet_data = True self._params['traffic'] = self.traffic_defaults.copy() if traffic: self._params['traffic'] = merge_spec( |