xena_verify: Add final verification option to Xena

Adds final verification option to Xena traffic generator
inside VSPerf. Works by running the Xena2544.exe again
after the initial run is complete with modified values
inside the json file. If the final verify fails it will
revert the settings and continue the in a pattern based
on if the smart search is enabled.

JIRA: VSPERF-506

Change-Id: Ic272628c489b88989f67549bd020af7354dc9b91
Signed-off-by: Christian Trautman <ctrautma@redhat.com>

4 files changed, 125 insertions, 3 deletions

diff --git a/conf/03_traffic.conf b/conf/03_traffic.conf
index db704d02..764fd731 100644
--- a/conf/03_traffic.conf
+++ b/conf/03_traffic.conf
@@ -386,6 +386,14 @@ TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION = '0.5'
 TRAFFICGEN_XENA_2544_TPUT_USEPASS_THRESHHOLD = 'false'
 TRAFFICGEN_XENA_2544_TPUT_PASS_THRESHHOLD = '0.0'
 
+# Xena RFC 2544 final verification options
+TRAFFICGEN_XENA_RFC2544_VERIFY = False
+TRAFFICGEN_XENA_RFC2544_VERIFY_DURATION = 120 # in seconds
+# Number of verify attempts before giving up...
+TRAFFICGEN_XENA_RFC2544_MAXIMUM_VERIFY_ATTEMPTS = 10
+# Logic for restarting binary search, see documentation for details
+TRAFFICGEN_XENA_RFC2544_BINARY_RESTART_SMART_SEARCH = True
+
 # Xena Continuous traffic options
 # Please reference xena documentation before making changes to these settings
 TRAFFICGEN_XENA_CONT_PORT_LEARNING_ENABLED = True
diff --git a/conf/10_custom.conf b/conf/10_custom.conf
index 819d2c32..1d62c0ce 100644
--- a/conf/10_custom.conf
+++ b/conf/10_custom.conf
@@ -88,6 +88,14 @@ TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION = '0.5'
 TRAFFICGEN_XENA_2544_TPUT_USEPASS_THRESHHOLD = 'false'
 TRAFFICGEN_XENA_2544_TPUT_PASS_THRESHHOLD = '0.0'
 
+# Xena RFC 2544 final verification options
+TRAFFICGEN_XENA_RFC2544_VERIFY = False
+TRAFFICGEN_XENA_RFC2544_VERIFY_DURATION = 120 # in seconds
+# Number of verify attempts before giving up...
+TRAFFICGEN_XENA_RFC2544_MAXIMUM_VERIFY_ATTEMPTS = 10
+# Logic for restarting binary search, see documentation for details
+TRAFFICGEN_XENA_RFC2544_BINARY_RESTART_SMART_SEARCH = True
+
 # Xena Continuous traffic options
 # Please reference xena documentation before making changes to these settings
 TRAFFICGEN_XENA_CONT_PORT_LEARNING_ENABLED = True
diff --git a/docs/testing/user/configguide/trafficgen.rst b/docs/testing/user/configguide/trafficgen.rst
index fff8af07..7a488ac9 100644
--- a/docs/testing/user/configguide/trafficgen.rst
+++ b/docs/testing/user/configguide/trafficgen.rst
@@ -606,6 +606,37 @@ Each value modifies the behavior of rfc 2544 throughput testing. Refer to your
 Xena documentation to understand the behavior changes in modifying these
 values.
 
+Xena RFC2544 testing inside VSPerf also includes a final verification option.
+This option allows for a faster binary search with a longer final verification
+of the binary search result. This feature can be enabled in the configuration
+files as well as the length of the final verification in seconds.
+
+..code-block:: python
+
+    TRAFFICGEN_XENA_RFC2544_VERIFY = False
+    TRAFFICGEN_XENA_RFC2544_VERIFY_DURATION = 120
+
+If the final verification does not pass the test with the lossrate specified
+it will continue the binary search from its previous point. If the smart search
+option is enabled the search will continue by taking the current pass rate minus
+the minimum and divided by 2. The maximum is set to the last pass rate minus the
+threshold value set.
+
+For example if the settings are as follows
+
+..code-block:: python
+
+    TRAFFICGEN_XENA_RFC2544_BINARY_RESTART_SMART_SEARCH = True
+    TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE = '0.5'
+    TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION = '0.5'
+
+and the verification attempt was 64.5, smart search would take 64.5 - 0.5 / 2.
+This would continue the search at 32 but still have a maximum possible value of
+64.
+
+If smart is not enabled it will just resume at the last pass rate minus the
+threshold value.
+
 Continuous Traffic Testing
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/tools/pkt_gen/xena/xena.py b/tools/pkt_gen/xena/xena.py
index c6b26f88..2601a7c3 100755
--- a/tools/pkt_gen/xena/xena.py
+++ b/tools/pkt_gen/xena/xena.py
@@ -366,7 +366,7 @@ class Xena(ITrafficGenerator):
         settings.setValue('XENA_VERSION', 'XENA Socket API - {}'.format(
             self.xmanager.get_version()))
 
-        if not len(self.xmanager.ports):
+        if not self.xmanager.ports:
             self.xmanager.ports[0] = self.xmanager.add_module_port(
                 settings.getValue('TRAFFICGEN_XENA_MODULE1'),
                 settings.getValue('TRAFFICGEN_XENA_PORT1'))
@@ -435,8 +435,8 @@ class Xena(ITrafficGenerator):
             port.set_port_ping_reply(is_on=True)
             port.set_port_ping_reply(is_on=True, ipv6=True)
 
-            stream.set_rate_fraction(
-                10000 * self._params['traffic']['frame_rate'])
+            stream.set_rate_fraction(int(
+                10000 * self._params['traffic']['frame_rate']))
             stream.set_packet_header(self._build_packet_header(
                 reverse=flip_addr))
             stream.set_header_protocol(
@@ -632,6 +632,81 @@ class Xena(ITrafficGenerator):
         self._wait_xena_2544_complete()
 
         root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
+
+        if settings.getValue('TRAFFICGEN_XENA_RFC2544_VERIFY'):
+            # record the previous settings so we can revert to them if needed to
+            # run the binary search again if the verify fails.
+            old_tests = tests
+            old_duration = self._duration
+            old_min = settings.getValue('TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE')
+
+            for attempt in range(
+                    1, settings.getValue(
+                        'TRAFFICGEN_XENA_RFC2544_MAXIMUM_VERIFY_ATTEMPTS')+1):
+                self._logger.info('Running verify attempt %s', attempt)
+                # get the last pass tx rate from the binary search
+                pass_rate = float(root[0][1][0].get('TotalTxRatePcnt'))
+                # run a one pass rfc2544 with the pass rate to see if it passes
+                # the verify duration
+                settings.setValue(
+                    'TRAFFICGEN_XENA_2544_TPUT_INIT_VALUE', pass_rate)
+                settings.setValue(
+                    'TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE', pass_rate)
+                settings.setValue(
+                    'TRAFFICGEN_XENA_2544_TPUT_MAX_VALUE', pass_rate)
+                self.start_rfc2544_throughput(
+                    traffic, 1, settings.getValue(
+                        'TRAFFICGEN_XENA_RFC2544_VERIFY_DURATION'), lossrate)
+                self.wait_rfc2544_throughput()
+                root = ET.parse(
+                    r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
+
+                # If it passed, report the number of lost frames and exit the
+                # loop
+                if root[0][1][0].get('TestState') == "PASS":
+                    self._logger.info('Verify passed, packets lost = %s',
+                                      root[0][1][0].get('TotalLossFrames'))
+                    break
+                elif attempt < settings.getValue(
+                        'TRAFFICGEN_XENA_RFC2544_MAXIMUM_VERIFY_ATTEMPTS'):
+                    self._logger.info(
+                        'Verify failed, resuming binary search, packets lost = %s',
+                        root[0][1][0].get('TotalLossFrames'))
+                    settings.setValue(
+                        'TRAFFICGEN_XENA_2544_TPUT_MAX_VALUE',
+                        pass_rate - float(settings.getValue(
+                            'TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION')))
+                    if settings.getValue(
+                            'TRAFFICGEN_XENA_RFC2544_BINARY_RESTART_SMART_SEARCH'):
+                        settings.setValue(
+                            'TRAFFICGEN_XENA_2544_TPUT_INIT_VALUE',
+                            (pass_rate - float(old_min)) / 2)
+                    else:
+                        settings.setValue(
+                            'TRAFFICGEN_XENA_2544_TPUT_INIT_VALUE',
+                            pass_rate - float(settings.getValue(
+                                'TRAFFICGEN_XENA_2544_TPUT_VALUE_RESOLUTION')))
+                    settings.setValue(
+                        'TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE', old_min)
+                    self._logger.debug(
+                        'RFC2544 Initial rate: %s',
+                        settings.getValue('TRAFFICGEN_XENA_2544_TPUT_INIT_VALUE'))
+                    self._logger.debug(
+                        'RFC2544 Maximum rate: %s',
+                        settings.getValue('TRAFFICGEN_XENA_2544_TPUT_MAX_VALUE'))
+                    self._logger.debug(
+                        'RFC2544 Minimum rate: %s',
+                        settings.getValue('TRAFFICGEN_XENA_2544_TPUT_MIN_VALUE'))
+                    self._duration = old_duration
+                    self.start_rfc2544_throughput(
+                        traffic, old_tests, self._duration, lossrate)
+                    self.wait_rfc2544_throughput()
+                    root = ET.parse(
+                        r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
+                else:
+                    self._logger.error(
+                        'Maximum number of verify attempts reached. Reporting last result')
+
         return Xena._create_throughput_result(root)
 
     def start_rfc2544_throughput(self, traffic=None, tests=1, duration=20,