Results: Default latency-histrogram with Spirent TestCenter

The patch adds support to provide Latency histogram from Spirent traffic
generator.
1. 03_traffic.conf: Enable histogram, and set type (default)
2. testcenter.py: If histogram enabled, call the script with right flag.
3. testcenter-rfc2544-rest.py: configure and write histogram to a
separte file in the default results folder.
4. Fix PyLint Errors
5. Adding a patch to test the 'build-error-fix'. Increased the
MAX_MEMSEG of in DPDK-config to 1024
6. Adding MAX_MEMSEG configuration at common_base didn't work.
7. Included packet sizes in the output.
8. Included description under spirent TGen
9. Removed MAX_MEMSEG configuration.

Change-Id: I7787c1768d7ac650f0ce581f17ec78df7a964e31
JIRA: VSPERF-522
Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>

2 files changed, 101 insertions, 5 deletions

diff --git a/tools/pkt_gen/testcenter/testcenter-rfc2544-rest.py b/tools/pkt_gen/testcenter/testcenter-rfc2544-rest.py
index 6c30b130..1ed12968 100644
--- a/tools/pkt_gen/testcenter/testcenter-rfc2544-rest.py
+++ b/tools/pkt_gen/testcenter/testcenter-rfc2544-rest.py
@@ -24,7 +24,7 @@ TestCenter REST APIs. This test supports Python 3.4
 import argparse
 import logging
 import os
-
+import sqlite3
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -39,6 +39,17 @@ def create_dir(path):
             raise
 
 
+def write_histogram_to_csv(results_path, csv_results_file_prefix,
+                           counts, ranges):
+    """ Write the results of the query to the CSV """
+    filec = os.path.join(results_path, csv_results_file_prefix + ".csv")
+    with open(filec, "wb") as result_file:
+        for key in counts:
+            result_file.write(str(key) + "\n")
+            result_file.write(str(ranges) + "\n")
+            result_file.write(str(counts[key]) + "\n")
+
+
 def write_query_results_to_csv(results_path, csv_results_file_prefix,
                                query_results):
     """ Write the results of the query to the CSV """
@@ -68,7 +79,8 @@ def percent_float(value):
             "%s not in range [0.0, 100.0]" % pvalue)
     return pvalue
 
-# pylint: disable=too-many-branches, too-many-statements
+
+# pylint: disable=too-many-branches, too-many-statements, too-many-locals
 def main():
     """ Read the arguments, Invoke Test and Return the results"""
     parser = argparse.ArgumentParser()
@@ -146,6 +158,11 @@ def main():
                                 default="./Results",
                                 help="The directory to copy results to",
                                 dest="results_dir")
+    optional_named.add_argument("--vsperf_results_dir",
+                                required=False,
+                                default="./Results",
+                                help="The directory to copy results to",
+                                dest="vsperf_results_dir")
     optional_named.add_argument("--csv_results_file_prefix",
                                 required=False,
                                 default="Rfc2544Tput",
@@ -269,6 +286,11 @@ def main():
                                       "the first emulated device interface"
                                       "on the first west port"),
                                 dest="west_intf_gateway_addr")
+    optional_named.add_argument("--latency_histogram",
+                                required=False,
+                                action="store_true",
+                                help="latency histogram is required in output?",
+                                dest="latency_histogram")
     parser.add_argument("-v",
                         "--verbose",
                         required=False,
@@ -309,6 +331,7 @@ def main():
         _LOGGER.debug("SpirentTestCenter system version: %s",
                       stc.get("system1", "version"))
 
+    # pylint: disable=too-many-nested-blocks
     try:
         device_list = []
         port_list = []
@@ -353,7 +376,9 @@ def main():
                                                         east_chassis_port})
         # Create the DeviceGenEthIIIfParams object
         stc.create("DeviceGenEthIIIfParams",
-                   under=east_device_gen_params)
+                   under=east_device_gen_params,
+                   attributes={'UseDefaultPhyMac':True})
+
         # Configuring Ipv4 interfaces
         stc.create("DeviceGenIpv4IfParams",
                    under=east_device_gen_params,
@@ -374,7 +399,9 @@ def main():
                                                         west_chassis_port})
         # Create the DeviceGenEthIIIfParams object
         stc.create("DeviceGenEthIIIfParams",
-                   under=west_device_gen_params)
+                   under=west_device_gen_params,
+                   attributes={'UseDefaultPhyMac':True})
+
         # Configuring Ipv4 interfaces
         stc.create("DeviceGenIpv4IfParams",
                    under=west_device_gen_params,
@@ -390,6 +417,32 @@ def main():
         if args.verbose:
             _LOGGER.debug(device_list)
 
+        # Configure Histogram
+        if args.latency_histogram:
+            # Generic Configuration
+            histResOptions = stc.get("project1", 'children-ResultOptions')
+            stc.config(histResOptions, {'ResultViewMode': 'HISTOGRAM'})
+            # East Port Configuration
+            histAnaEast = stc.get(east_chassis_port, 'children-Analyzer')
+            histAnaEastConfig = stc.get(histAnaEast, 'children-AnalyzerConfig')
+            stc.config(histAnaEastConfig, {'HistogramMode': 'LATENCY'})
+            eLatHist = stc.get(histAnaEastConfig, 'children-LatencyHistogram')
+            stc.config(eLatHist, {'ConfigMode': 'CONFIG_LIMIT_MODE',
+                                  'BucketSizeUnit': 'ten_nanoseconds',
+                                  'Active': 'TRUE',
+                                  'DistributionMode': 'CENTERED_MODE'})
+            # West Port Configuration
+            histAnaWest = stc.get(west_chassis_port, 'children-Analyzer')
+            histAnaWestConfig = stc.get(histAnaWest, 'children-AnalyzerConfig')
+            stc.config(histAnaWestConfig, {'HistogramMode': 'LATENCY'})
+            wLatHist = stc.get(histAnaWestConfig, 'children-LatencyHistogram')
+            stc.config(wLatHist, {'ConfigMode': 'CONFIG_LIMIT_MODE',
+                                  'BucketSizeUnit': 'ten_nanoseconds',
+                                  'Active': 'TRUE',
+                                  'DistributionMode': 'CENTERED_MODE'})
+            gBucketSizeList = stc.get(wLatHist, 'BucketSizeList')
+            # gLimitSizeList  = stc.get(wLatHist, 'LimitList')
+
         # Create the RFC 2544 'metric test
         if args.metric == "throughput":
             if args.verbose:
@@ -485,6 +538,42 @@ def main():
             _LOGGER.debug("The lab server results database is %s",
                           lab_server_resultsdb)
 
+        # Create Latency Histogram CSV file()
+        if args.latency_histogram:
+            hist_dict_counts = {}
+            for file_url in stc.files():
+                if '-FrameSize-' in file_url:
+                    stc.download(file_url)
+                    filename = file_url.split('/')[-1]
+                    if os.path.exists(os.getcwd() + '/' + filename):
+                        conn = sqlite3.connect(os.getcwd() + '/' + filename)
+                        # cursor = conn.execute(
+                        #    'select * from RxEotStreamResults')
+                        # names = [desc[0] for desc in cursor.description]
+                        counts = conn.execute("SELECT \
+                                              HistBin1Count, HistBin2Count,\
+                                              HistBin3Count, HistBin4Count,\
+                                              HistBin5Count, HistBin6Count,\
+                                              HistBin7Count, HistBin8Count,\
+                                              HistBin9Count, HistBin10Count,\
+                                              HistBin11Count, HistBin12Count,\
+                                              HistBin13Count, HistBin14Count, \
+                                              HistBin15Count, HistBin16Count \
+                                              from RxEotStreamResults")
+                        strs = filename.split('-')
+                        key = strs[strs.index('FrameSize')+1]
+                        if key in hist_dict_counts:
+                            hist_dict_counts[key] = [a+b for a, b in
+                                                     zip(counts.fetchone(),
+                                                         hist_dict_counts[key])]
+                        else:
+                            hist_dict_counts[key] = counts.fetchone()
+                        conn.close()
+
+            write_histogram_to_csv(args.vsperf_results_dir, 'Histogram',
+                                   hist_dict_counts,
+                                   gBucketSizeList)
+
         stc.perform("CSSynchronizeFiles",
                     params={"DefaultDownloadDir": args.results_dir})
 
diff --git a/tools/pkt_gen/testcenter/testcenter.py b/tools/pkt_gen/testcenter/testcenter.py
index 487566bf..7afa3d8d 100644
--- a/tools/pkt_gen/testcenter/testcenter.py
+++ b/tools/pkt_gen/testcenter/testcenter.py
@@ -98,7 +98,9 @@ def get_rfc2544_common_settings():
             "--trial_duration_sec",
             settings.getValue("TRAFFICGEN_STC_TRIAL_DURATION_SEC"),
             "--traffic_pattern",
-            settings.getValue("TRAFFICGEN_STC_TRAFFIC_PATTERN")]
+            settings.getValue("TRAFFICGEN_STC_TRAFFIC_PATTERN"),
+            "--vsperf_results_dir",
+            settings.getValue("RESULTS_PATH")]
     return args
 
 
@@ -419,6 +421,11 @@ class TestCenter(trafficgen.ITrafficGenerator):
                                                           tests)
         args = rfc2544_common_args + stc_common_args + rfc2544_custom_args
 
+        if traffic and 'latency_histogram' in traffic:
+            if traffic['latency_histogram']['enabled']:
+                if traffic['latency_histogram']['type'] == 'Default':
+                    args.append("--latency_histogram")
+
         if settings.getValue("TRAFFICGEN_STC_VERBOSE") == "True":
             args.append("--verbose")
             verbose = True