NFVBENCH-177: Add a config item 'user_info' and theoretical max rate value

Change-Id: If96ccbffab67cfc0a08279d94cf7a5e81d958044
Signed-off-by: fmenguy <francoisregis.menguy@orange.com>

9 files changed, 232 insertions, 13 deletions

diff --git a/docs/testing/user/userguide/advanced.rst b/docs/testing/user/userguide/advanced.rst
index ba212c5..62d17b6 100644
--- a/docs/testing/user/userguide/advanced.rst
+++ b/docs/testing/user/userguide/advanced.rst
@@ -500,4 +500,50 @@ Check on the NFVBench window that the following log appears just before the test
     2019-10-21 09:38:51,532 INFO Running traffic generator
     2019-10-21 09:38:51,541 INFO ``Service mode is enabled``
     2019-10-21 09:38:52,552 INFO TX: 2004; RX: 2003; Est. Dropped: 1; Est. Drop rate: 0.0499%
-    2019-10-21 09:38:53,559 INFO TX: 4013; RX: 4011; Est. Dropped: 2; Est. Drop rate: 0.0498%
\ No newline at end of file
+    2019-10-21 09:38:53,559 INFO TX: 4013; RX: 4011; Est. Dropped: 2; Est. Drop rate: 0.0498%
+
+User info data
+--------------
+
+The ``--user-info`` option allows you to pass custom information as a JSON string.
+This information will be available through JSON output and also exported to ``fluentd`` and can be used in results post-processing.
+
+Example of use :
+
+.. code-block:: bash
+
+    nfvbench ``--user-info='{"status":"explore","description":{"target":"lab","ok":true,"version":2020}'``
+
+.. note:: only JSON string is allowed
+
+``--user-info`` can be used for determining theoretical max rate. In some cases, an overhead encapsulation exists between NFVbench and SUT so NFVbench will not reach line rate inside SUT due to this extra encapsulation.
+To calculate this theoretical line rate inside SUT, NFVbench will use a reserved key: ``extra_encapsulation_bytes`` in ``--user-info`` property.
+
+.. code-block:: bash
+
+    nfvbench ``--user-info='{"extra_encapsulation_bytes": 28}'``
+
+
+As a result, NFVbench will return two values ``theoretical_tx_rate_bps`` and ``theoretical_tx_rate_pps``:
+
+.. code-block:: bash
+
+                                "ndr": {
+                                    "duration_sec": 2.0,
+                                    "initial_rate_type": "rate_percent",
+                                    "l2frame_size": "64",
+                                    "load_percent_per_direction": 100.0,
+                                    "rate_bps": 20000000000.0,
+                                    "rate_percent": 200.0,
+                                    "rate_pps": 29761904,
+                                    "stats": {
+                                        ...
+                                        "offered_tx_rate_bps": 15000000000.0,
+                                        ...
+                                        "theoretical_tx_rate_bps": 15000000000.0,
+                                        "theoretical_tx_rate_pps": 22321428.57142857,
+                                        "total_tx_rate": 22321428
+                                    },
+
+In the above example, line rate is 20Gbps but NFVbench is outside SUT and a SDN gateway add an extra encapsulation of 28 bytes.
+Overall, theoretical line rate inside SUT is only 15 Gbps for 64 bytes packet size and it will be this max capacity treated by the target compute node.
diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml
index 253e8bc..b33d02c 100755
--- a/nfvbench/cfg.default.yaml
+++ b/nfvbench/cfg.default.yaml
@@ -331,6 +331,26 @@ restart: false
 # if empty defaults to the one specified in generator_profile.cores
 cores:
 
+# Simpler override for the interface speed
+# if empty, the current generator_profile.intf_speed parameter applies
+# if value = 'auto' the auto-detection is forced
+intf_speed:
+
+# 'cores' and 'intf_speed' parameters can be overriden themselves
+# by respective options --cores and --intf-speed on the command-line.
+
+# By default, the real ports line rate is detected and used as
+# the reference for computing the theoretical maximum traffic load (100%).
+# Note that specifying 'intf_speed' allows to artificially lower this
+# reference while not modifying the actual transmission bit rate.
+
+# The values of the following parameters are ignored on entry
+# they are defined here in order to appear in the reported configuration.
+# They will reflect the value active at run-time (after overriding or detection)
+cores_used:
+intf_speed_used:
+intf_speed_detected:
+
 # Add cache size in packet generation for TRex field engine (FE).
 # More information for TRex performance:
 # https://trex-tgn.cisco.com/trex/doc/trex_stateless.html#_tutorial_field_engine_significantly_improve_performance
@@ -338,6 +358,8 @@ cores:
 # If cache_size < 0: cache_size will be set to flow count value
 cache_size: 0
 # The cache size is actually limited by the number of 64B mbufs configured in the trex platform configuration (see Trex manual 6.2.2. Memory section configuration)
+# Note that the resulting value is finally clipped to 10000, whatever the requested size is (by design limitation).
+
 # Trex will use 1 x 64B mbuf per pre-built cached packet, assuming 1 pre-built cached packet per flow, it means for very large number of flows, the number of configured mbuf_64 will need to be set accordingly.
 mbuf_64:
 
@@ -808,15 +830,28 @@ factory_class: 'BasicFactory'
 # Can be overriden by --user-label
 user_label:
 
-
-# THESE FIELDS SHOULD BE USED VERY RARELY
+# Custom information to be passed to results post-processing,
+# they will be included as is in the json report 'config' branch.
+# Useful for documenting or automating further treatments.
+# The value is any yaml object (=> open usage) - example:
+# |user_info:
+# |    status: explore
+# |    description:
+# |        generator: VM
+# |        attachment: direct
+# |        target: lab-pf
+# |        switch: qfx3500
+# Keys may be merged/overriden using the --user-info command line option
+# (the command-line parameter value is expressed as a json object string)
+user_info:
+
+# THESE FIELDS SHOULD BE USED VERY RARELY OR ON PURPOSE
 
 # Skip vswitch configuration and retrieving of stats
 # Can be overriden by --no-vswitch-access
 # Should be left to the default value (false)
 no_vswitch_access: false
 
-
 # Enable service mode for trafic capture from TRex console (for debugging purpose)
 # Can be overriden by --service-mode
 # Should be left to the default value (false)
diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py
index 651d06b..9b8b3d1 100644
--- a/nfvbench/nfvbench.py
+++ b/nfvbench/nfvbench.py
@@ -274,6 +274,23 @@ class NFVBench(object):
         self.config_plugin.validate_config(config, self.specs.openstack)
 
 
+def bool_arg(x):
+    """Argument type to be used in parser.add_argument()
+    When a boolean like value is expected to be given
+    """
+    return (str(x).lower() != 'false') \
+        and (str(x).lower() != 'no') \
+        and (str(x).lower() != '0')
+
+
+def int_arg(x):
+    """Argument type to be used in parser.add_argument()
+    When an integer type value is expected to be given
+    (returns 0 if argument is invalid, hexa accepted)
+    """
+    return int(x, 0)
+
+
 def _parse_opts_from_cli():
     parser = argparse.ArgumentParser()
 
@@ -476,6 +493,39 @@ def _parse_opts_from_cli():
                         metavar='<vlan>',
                         help='Port to port or port to switch to port L2 loopback with VLAN id')
 
+    """Option to allow for passing custom information to results post-processing"""
+    parser.add_argument('--user-info', dest='user_info',
+                        action='store',
+                        metavar='<data>',
+                        help='Custom data to be included as is in the json report config branch - '
+                                + ' example, pay attention! no space: '
+                                + '--user-info=\'{"status":"explore","description":{"target":"lab"'
+                                + ',"ok":true,"version":2020}\'')
+
+    """Option to allow for overriding the NFVbench 'vlan_tagging' option"""
+    parser.add_argument('--vlan-tagging', dest='vlan_tagging',
+                        type=bool_arg,
+                        metavar='<boolean>',
+                        action='store',
+                        default=None,
+                        help='Override the NFVbench \'vlan_tagging\' parameter')
+
+    """Option to allow for overriding the T-Rex 'intf_speed' parameter"""
+    parser.add_argument('--intf-speed', dest='intf_speed',
+                        metavar='<speed>',
+                        action='store',
+                        default=None,
+                        help='Override the NFVbench \'intf_speed\' parameter '
+                                + '(e.g. 10Gbps, auto, 16.72Gbps)')
+
+    """Option to allow for overriding the T-Rex 'cores' parameter"""
+    parser.add_argument('--cores', dest='cores',
+                        type=int_arg,
+                        metavar='<number>',
+                        action='store',
+                        default=None,
+                        help='Override the T-Rex \'cores\' parameter')
+
     parser.add_argument('--cache-size', dest='cache_size',
                         action='store',
                         default='0',
@@ -600,7 +650,8 @@ def main():
         config.name = ''
         if opts.config:
             # do not check extra_specs in flavor as it can contain any key/value pairs
-            whitelist_keys = ['extra_specs']
+            # the same principle applies also to the optional user_info open property
+            whitelist_keys = ['extra_specs', 'user_info']
             # override default config options with start config at path parsed from CLI
             # check if it is an inline yaml/json config or a file name
             if os.path.isfile(opts.config):
@@ -619,6 +670,17 @@ def main():
                 LOG.addHandler(fluent_logger)
                 break
 
+        # convert 'user_info' opt from json string to dictionnary
+        # and merge the result with the current config dictionnary
+        if opts.user_info:
+            opts.user_info = json.loads(opts.user_info)
+            if config.user_info:
+                config.user_info = config.user_info + opts.user_info
+            else:
+                config.user_info = opts.user_info
+            # hide the option to further _update_config()
+            opts.user_info = None
+
         # traffic profile override options
         override_custom_traffic(config, opts.frame_sizes, opts.unidir)
 
diff --git a/nfvbench/summarizer.py b/nfvbench/summarizer.py
index 326de10..bbd5908 100644
--- a/nfvbench/summarizer.py
+++ b/nfvbench/summarizer.py
@@ -421,6 +421,8 @@ class NFVBenchSummarizer(Summarizer):
                     'rate_bps': analysis['ndr']['rate_bps'],
                     'rate_pps': analysis['ndr']['rate_pps'],
                     'offered_tx_rate_bps': analysis['ndr']['stats']['offered_tx_rate_bps'],
+                    'theoretical_tx_rate_pps': analysis['ndr']['stats']['theoretical_tx_rate_pps'],
+                    'theoretical_tx_rate_bps': analysis['ndr']['stats']['theoretical_tx_rate_bps'],
                     'drop_percentage': analysis['ndr']['stats']['overall']['drop_percentage'],
                     'avg_delay_usec': analysis['ndr']['stats']['overall']['avg_delay_usec'],
                     'min_delay_usec': analysis['ndr']['stats']['overall']['min_delay_usec'],
@@ -455,6 +457,8 @@ class NFVBenchSummarizer(Summarizer):
                     'rate_bps': analysis['pdr']['rate_bps'],
                     'rate_pps': analysis['pdr']['rate_pps'],
                     'offered_tx_rate_bps': analysis['pdr']['stats']['offered_tx_rate_bps'],
+                    'theoretical_tx_rate_pps': analysis['pdr']['stats']['theoretical_tx_rate_pps'],
+                    'theoretical_tx_rate_bps': analysis['pdr']['stats']['theoretical_tx_rate_bps'],
                     'drop_percentage': analysis['pdr']['stats']['overall']['drop_percentage'],
                     'avg_delay_usec': analysis['pdr']['stats']['overall']['avg_delay_usec'],
                     'min_delay_usec': analysis['pdr']['stats']['overall']['min_delay_usec'],
@@ -480,6 +484,8 @@ class NFVBenchSummarizer(Summarizer):
                 single_run_data = {
                     'type': 'single_run',
                     'offered_tx_rate_bps': analysis['stats']['offered_tx_rate_bps'],
+                    'theoretical_tx_rate_pps': analysis['stats']['theoretical_tx_rate_pps'],
+                    'theoretical_tx_rate_bps': analysis['stats']['theoretical_tx_rate_bps'],
                     'drop_rate_percent': analysis['stats']['overall']['drop_rate_percent'],
                     'avg_delay_usec': analysis['stats']['overall']['rx']['avg_delay_usec'],
                     'min_delay_usec': analysis['stats']['overall']['rx']['min_delay_usec'],
diff --git a/nfvbench/traffic_client.py b/nfvbench/traffic_client.py
index e2ae977..72fbb5d 100755
--- a/nfvbench/traffic_client.py
+++ b/nfvbench/traffic_client.py
@@ -486,15 +486,24 @@ class GeneratorConfig(object):
             self.cores = config.cores
         else:
             self.cores = gen_config.get('cores', 1)
+        # let's report the value actually used in the end
+        config.cores_used = self.cores
         self.mbuf_factor = config.mbuf_factor
         self.mbuf_64 = config.mbuf_64
         self.hdrh = not config.disable_hdrh
-        if gen_config.intf_speed:
-            # interface speed is overriden from config
-            self.intf_speed = bitmath.parse_string(gen_config.intf_speed.replace('ps', '')).bits
+        if config.intf_speed:
+            # interface speed is overriden from the command line
+            self.intf_speed = config.intf_speed
+        elif gen_config.intf_speed:
+            # interface speed is overriden from the generator config
+            self.intf_speed = gen_config.intf_speed
         else:
+            self.intf_speed = "auto"
+        if self.intf_speed == "auto" or self.intf_speed == "0":
             # interface speed is discovered/provided by the traffic generator
             self.intf_speed = 0
+        else:
+            self.intf_speed = bitmath.parse_string(self.intf_speed.replace('ps', '')).bits
         self.name = gen_config.name
         self.zmq_pub_port = gen_config.get('zmq_pub_port', 4500)
         self.zmq_rpc_port = gen_config.get('zmq_rpc_port', 4501)
@@ -713,13 +722,17 @@ class TrafficClient(object):
             # interface speed is overriden from config
             if self.intf_speed != tg_if_speed:
                 # Warn the user if the speed in the config is different
-                LOG.warning('Interface speed provided is different from actual speed (%d Gbps)',
-                            intf_speeds[0])
+                LOG.warning(
+                    'Interface speed provided (%g Gbps) is different from actual speed (%d Gbps)',
+                    self.intf_speed / 1000000000.0, intf_speeds[0])
         else:
             # interface speed not provisioned by config
             self.intf_speed = tg_if_speed
             # also update the speed in the tg config
             self.generator_config.intf_speed = tg_if_speed
+        # let's report detected and actually used interface speed
+        self.config.intf_speed_detected = tg_if_speed
+        self.config.intf_speed_used = self.intf_speed
 
         # Save the traffic generator local MAC
         for mac, device in zip(self.gen.get_macs(), self.generator_config.devices):
@@ -923,7 +936,9 @@ class TrafficClient(object):
         """Collect final stats for previous run."""
         stats = self.gen.get_stats(self.ifstats)
         retDict = {'total_tx_rate': stats['total_tx_rate'],
-                   'offered_tx_rate_bps': stats['offered_tx_rate_bps']}
+                   'offered_tx_rate_bps': stats['offered_tx_rate_bps'],
+                   'theoretical_tx_rate_bps': stats['theoretical_tx_rate_bps'],
+                   'theoretical_tx_rate_pps': stats['theoretical_tx_rate_pps']}
 
         tx_keys = ['total_pkts', 'total_pkt_bytes', 'pkt_rate', 'pkt_bit_rate']
         rx_keys = tx_keys + ['dropped_pkts']
diff --git a/nfvbench/traffic_gen/dummy.py b/nfvbench/traffic_gen/dummy.py
index 25664e5..8a6d11a 100644
--- a/nfvbench/traffic_gen/dummy.py
+++ b/nfvbench/traffic_gen/dummy.py
@@ -151,6 +151,8 @@ class DummyTG(AbstractTrafficGenerator):
         avg_packet_size = utils.get_average_packet_size(self.l2_frame_size)
         total_tx_bps = utils.pps_to_bps(total_tx_pps, avg_packet_size)
         result['offered_tx_rate_bps'] = total_tx_bps
+
+        result.update(self.get_theoretical_rates(avg_packet_size))
         return result
 
     def get_stream_stats(self, tg_stats, if_stats, latencies, chain_idx):
diff --git a/nfvbench/traffic_gen/traffic_base.py b/nfvbench/traffic_gen/traffic_base.py
index abf5a22..df28772 100644
--- a/nfvbench/traffic_gen/traffic_base.py
+++ b/nfvbench/traffic_gen/traffic_base.py
@@ -15,6 +15,8 @@
 import abc
 import sys
 
+import bitmath
+
 from nfvbench.log import LOG
 from . import traffic_utils
 
@@ -126,3 +128,32 @@ class AbstractTrafficGenerator(object):
 
         return: a list of speed in Gbps indexed by the port#
         """
+
+    def get_theoretical_rates(self, avg_packet_size):
+
+        result = {}
+
+        intf_speeds = self.get_port_speed_gbps()
+        tg_if_speed = bitmath.parse_string(str(intf_speeds[0]) + 'Gb').bits
+        intf_speed = tg_if_speed
+
+        if hasattr(self.config, 'intf_speed') and self.config.intf_speed is not None:
+            # in case of limitation due to config, TG speed is not accurate
+            # value is overridden by conf
+            if self.config.intf_speed != tg_if_speed:
+                intf_speed = bitmath.parse_string(self.config.intf_speed.replace('ps', '')).bits
+
+        if hasattr(self.config, 'user_info') and self.config.user_info is not None:
+            if "extra_encapsulation_bytes" in self.config.user_info:
+                frame_size_full_encapsulation = avg_packet_size + self.config.user_info[
+                    "extra_encapsulation_bytes"]
+                result['theoretical_tx_rate_pps'] = traffic_utils.bps_to_pps(
+                    intf_speed, frame_size_full_encapsulation) * 2
+                result['theoretical_tx_rate_bps'] = traffic_utils.pps_to_bps(
+                    result['theoretical_tx_rate_pps'], avg_packet_size)
+        else:
+            result['theoretical_tx_rate_pps'] = traffic_utils.bps_to_pps(intf_speed,
+                                                                         avg_packet_size) * 2
+            result['theoretical_tx_rate_bps'] = traffic_utils.pps_to_bps(
+                result['theoretical_tx_rate_pps'], avg_packet_size)
+        return result
diff --git a/nfvbench/traffic_gen/trex_gen.py b/nfvbench/traffic_gen/trex_gen.py
index 0bf6d93..f7250da 100644
--- a/nfvbench/traffic_gen/trex_gen.py
+++ b/nfvbench/traffic_gen/trex_gen.py
@@ -159,6 +159,9 @@ class TRex(AbstractTrafficGenerator):
         avg_packet_size = utils.get_average_packet_size(self.l2_frame_size)
         total_tx_bps = utils.pps_to_bps(result["total_tx_rate"], avg_packet_size)
         result['offered_tx_rate_bps'] = total_tx_bps
+
+        result.update(self.get_theoretical_rates(avg_packet_size))
+
         result["flow_stats"] = in_stats["flow_stats"]
         result["latency"] = in_stats["latency"]
 
diff --git a/test/test_nfvbench.py b/test/test_nfvbench.py
index 5274557..fe8742f 100644
--- a/test/test_nfvbench.py
+++ b/test/test_nfvbench.py
@@ -466,11 +466,12 @@ def _get_dummy_tg_config(chain_type, rate, scc=1, fc=10, step_ip='0.0.0.1',
         'service_mode': False,
         'no_flow_stats': False,
         'no_latency_stats': False,
-        'no_latency_streams': False
+        'no_latency_streams': False,
+        'intf_speed': '10Gbps'
 
     })
 
-def _get_traffic_client():
+def _get_traffic_client(user_info=None):
     config = _get_dummy_tg_config('PVP', 'ndr_pdr')
     config['vxlan'] = False
     config['mpls'] = False
@@ -478,6 +479,8 @@ def _get_traffic_client():
     config['pdr_run'] = True
     config['generator_profile'] = 'dummy'
     config['single_run'] = False
+    if user_info:
+        config['user_info'] = user_info
     traffic_client = TrafficClient(config)
     traffic_client.start_traffic_generator()
     traffic_client.set_traffic('64', True)
@@ -538,6 +541,22 @@ def test_ndr_pdr_low_cpu():
     # pp.pprint(results)
 
 @patch.object(TrafficClient, 'skip_sleep', lambda x: True)
+def test_ndr_at_lr_sdn_gw_encapsulation():
+    """Test NDR at line rate with traffic gen outside SUT and connected via SDN GW."""
+    user_info = {'extra_encapsulation_bytes': 28}
+    traffic_client = _get_traffic_client(user_info)
+    tg = traffic_client.gen
+    # this is a perfect sut with no loss at LR
+    tg.set_response_curve(lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100)
+    # tx packets should be line rate for 64B and no drops...
+    assert tg.get_tx_pps_dropped_pps(100) == (LR_64B_PPS, 0)
+    # NDR and PDR should be at 100%
+    # traffic_client.ensure_end_to_end()
+    results = traffic_client.get_ndr_and_pdr()
+    assert results['ndr']['stats']['theoretical_tx_rate_bps'] == 15000000000.0
+    assert_ndr_pdr(results, 200.0, 0.0, 200.0, 0.0)
+
+@patch.object(TrafficClient, 'skip_sleep', lambda x: True)
 def test_no_openstack():
     """Test nfvbench using main."""
     config = _get_dummy_tg_config('EXT', '1000pps')