Merge "xena_verify: Add final verification option to Xena"

4 files changed, 125 insertions, 3 deletions

diff --git a/conf/03_traffic.conf b/conf/03_traffic.conf
index db704d02..764fd731 100644
--- a/conf/03_traffic.conf
+++ b/conf/03_traffic.conf
@@ -386,6 +386,14 @@ TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION = '0.5'
 TRAFFICGEN_XENA_2544_TPUT_USEPASS_THRESHHOLD = 'false'
 TRAFFICGEN_XENA_2544_TPUT_PASS_THRESHHOLD = '0.0'
 
+# Xena RFC 2544 final verification options
+TRAFFICGEN_XENA_RFC2544_VERIFY = False
+TRAFFICGEN_XENA_RFC2544_VERIFY_DURATION = 120 # in seconds
+# Number of verify attempts before giving up...
+TRAFFICGEN_XENA_RFC2544_MAXIMUM_VERIFY_ATTEMPTS = 10
+# Logic for restarting binary search, see documentation for details
+TRAFFICGEN_XENA_RFC2544_BINARY_RESTART_SMART_SEARCH = True
+
 # Xena Continuous traffic options
 # Please reference xena documentation before making changes to these settings
 TRAFFICGEN_XENA_CONT_PORT_LEARNING_ENABLED = True
diff --git a/conf/10_custom.conf b/conf/10_custom.conf
index 819d2c32..1d62c0ce 100644
--- a/conf/10_custom.conf
+++ b/conf/10_custom.conf
@@ -88,6 +88,14 @@ TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION = '0.5'
 TRAFFICGEN_XENA_2544_TPUT_USEPASS_THRESHHOLD = 'false'
 TRAFFICGEN_XENA_2544_TPUT_PASS_THRESHHOLD = '0.0'
 
+# Xena RFC 2544 final verification options
+TRAFFICGEN_XENA_RFC2544_VERIFY = False
+TRAFFICGEN_XENA_RFC2544_VERIFY_DURATION = 120 # in seconds
+# Number of verify attempts before giving up...
+TRAFFICGEN_XENA_RFC2544_MAXIMUM_VERIFY_ATTEMPTS = 10
+# Logic for restarting binary search, see documentation for details
+TRAFFICGEN_XENA_RFC2544_BINARY_RESTART_SMART_SEARCH = True
+
 # Xena Continuous traffic options
 # Please reference xena documentation before making changes to these settings
 TRAFFICGEN_XENA_CONT_PORT_LEARNING_ENABLED = True
diff --git a/docs/testing/user/configguide/trafficgen.rst b/docs/testing/user/configguide/trafficgen.rst
index fff8af07..7a488ac9 100644
--- a/docs/testing/user/configguide/trafficgen.rst
+++ b/docs/testing/user/configguide/trafficgen.rst
@@ -606,6 +606,37 @@ Each value modifies the behavior of rfc 2544 throughput testing. Refer to your
 Xena documentation to understand the behavior changes in modifying these
 values.
 
+Xena RFC2544 testing inside VSPerf also includes a final verification option.
+This option allows for a faster binary search with a longer final verification
+of the binary search result. This feature can be enabled in the configuration
+files as well as the length of the final verification in seconds.
+
+..code-block:: python
+
+    TRAFFICGEN_XENA_RFC2544_VERIFY = False
+    TRAFFICGEN_XENA_RFC2544_VERIFY_DURATION = 120
+
+If the final verification does not pass the test with the lossrate specified
+it will continue the binary search from its previous point. If the smart search
+option is enabled the search will continue by taking the current pass rate minus
+the minimum and divided by 2. The maximum is set to the last pass rate minus the
+threshold value set.
+
+For example if the settings are as follows
+
+..code-block:: python
+
+    TRAFFICGEN_XENA_RFC2544_BINARY_RESTART_SMART_SEARCH = True
+    TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE = '0.5'
+    TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION = '0.5'
+
+and the verification attempt was 64.5, smart search would take 64.5 - 0.5 / 2.
+This would continue the search at 32 but still have a maximum possible value of
+64.
+
+If smart is not enabled it will just resume at the last pass rate minus the
+threshold value.
+
 Continuous Traffic Testing
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/tools/pkt_gen/xena/xena.py b/tools/pkt_gen/xena/xena.py
index c6b26f88..2601a7c3 100755
--- a/tools/pkt_gen/xena/xena.py
+++ b/tools/pkt_gen/xena/xena.py
@@ -366,7 +366,7 @@ class Xena(ITrafficGenerator):
         settings.setValue('XENA_VERSION', 'XENA Socket API - {}'.format(
             self.xmanager.get_version()))
 
-        if not len(self.xmanager.ports):
+        if not self.xmanager.ports:
             self.xmanager.ports[0] = self.xmanager.add_module_port(
                 settings.getValue('TRAFFICGEN_XENA_MODULE1'),
                 settings.getValue('TRAFFICGEN_XENA_PORT1'))
@@ -435,8 +435,8 @@ class Xena(ITrafficGenerator):
             port.set_port_ping_reply(is_on=True)
             port.set_port_ping_reply(is_on=True, ipv6=True)
 
-            stream.set_rate_fraction(
-                10000 * self._params['traffic']['frame_rate'])
+            stream.set_rate_fraction(int(
+                10000 * self._params['traffic']['frame_rate']))
             stream.set_packet_header(self._build_packet_header(
                 reverse=flip_addr))
             stream.set_header_protocol(
@@ -632,6 +632,81 @@ class Xena(ITrafficGenerator):
         self._wait_xena_2544_complete()
 
         root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
+
+        if settings.getValue('TRAFFICGEN_XENA_RFC2544_VERIFY'):
+            # record the previous settings so we can revert to them if needed to
+            # run the binary search again if the verify fails.
+            old_tests = tests
+            old_duration = self._duration
+            old_min = settings.getValue('TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE')
+
+            for attempt in range(
+                    1, settings.getValue(
+                        'TRAFFICGEN_XENA_RFC2544_MAXIMUM_VERIFY_ATTEMPTS')+1):
+                self._logger.info('Running verify attempt %s', attempt)
+                # get the last pass tx rate from the binary search
+                pass_rate = float(root[0][1][0].get('TotalTxRatePcnt'))
+                # run a one pass rfc2544 with the pass rate to see if it passes
+                # the verify duration
+                settings.setValue(
+                    'TRAFFICGEN_XENA_2544_TPUT_INIT_VALUE', pass_rate)
+                settings.setValue(
+                    'TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE', pass_rate)
+                settings.setValue(
+                    'TRAFFICGEN_XENA_2544_TPUT_MAX_VALUE', pass_rate)
+                self.start_rfc2544_throughput(
+                    traffic, 1, settings.getValue(
+                        'TRAFFICGEN_XENA_RFC2544_VERIFY_DURATION'), lossrate)
+                self.wait_rfc2544_throughput()
+                root = ET.parse(
+                    r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
+
+                # If it passed, report the number of lost frames and exit the
+                # loop
+                if root[0][1][0].get('TestState') == "PASS":
+                    self._logger.info('Verify passed, packets lost = %s',
+                                      root[0][1][0].get('TotalLossFrames'))
+                    break
+                elif attempt < settings.getValue(
+                        'TRAFFICGEN_XENA_RFC2544_MAXIMUM_VERIFY_ATTEMPTS'):
+                    self._logger.info(
+                        'Verify failed, resuming binary search, packets lost = %s',
+                        root[0][1][0].get('TotalLossFrames'))
+                    settings.setValue(
+                        'TRAFFICGEN_XENA_2544_TPUT_MAX_VALUE',
+                        pass_rate - float(settings.getValue(
+                            'TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION')))
+                    if settings.getValue(
+                            'TRAFFICGEN_XENA_RFC2544_BINARY_RESTART_SMART_SEARCH'):
+                        settings.setValue(
+                            'TRAFFICGEN_XENA_2544_TPUT_INIT_VALUE',
+                            (pass_rate - float(old_min)) / 2)
+                    else:
+                        settings.setValue(
+                            'TRAFFICGEN_XENA_2544_TPUT_INIT_VALUE',
+                            pass_rate - float(settings.getValue(
+                                'TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION')))
+                    settings.setValue(
+                        'TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE', old_min)
+                    self._logger.debug(
+                        'RFC2544 Initial rate: %s',
+                        settings.getValue('TRAFFICGEN_XENA_2544_TPUT_INIT_VALUE'))
+                    self._logger.debug(
+                        'RFC2544 Maximum rate: %s',
+                        settings.getValue('TRAFFICGEN_XENA_2544_TPUT_MAX_VALUE'))
+                    self._logger.debug(
+                        'RFC2544 Minimum rate: %s',
+                        settings.getValue('TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE'))
+                    self._duration = old_duration
+                    self.start_rfc2544_throughput(
+                        traffic, old_tests, self._duration, lossrate)
+                    self.wait_rfc2544_throughput()
+                    root = ET.parse(
+                        r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
+                else:
+                    self._logger.error(
+                        'Maximum number of verify attempts reached. Reporting last result')
+
         return Xena._create_throughput_result(root)
 
     def start_rfc2544_throughput(self, traffic=None, tests=1, duration=20,