Support for imix packet sizes

All packet size in the *.test files have been replaces with imix sizes.
imix is now a list of packet sizes. If you want to run with only a
pakcet size of 64, you need to specify a list containing only 1 size:
[64].
You can also specify multiple imix values by specifying a list of list
of sizes. The keyworkds in the test files are now imix & imixs.

Change-Id: Iea47c0266b022133b7bbfc9a6811e3199ae41521
Signed-off-by: Luc Provoost <luc.provoost@intel.com>

18 files changed, 69 insertions, 41 deletions

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput.test b/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput.test
index db7c9139..d931faa8 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput.test
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput.test
@@ -36,7 +36,7 @@ cores = [1]
 [test1]
 test=warmuptest
 flowsize=512
-packetsize=64
+imix=[64]
 warmupspeed=1
 warmuptime=2
 
@@ -47,7 +47,7 @@ test=TST009test
 # all combinations must be meeting the same threshold
 # The threshold is expressed in Mpps
 pass_threshold=0.1
-packetsizes=[64,128]
+imixs=[[64],[128]]
 # the number of flows in the list need to be powers of 2, max 2^20
 # Select from following numbers: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576
 flows=[8,1024]
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/bare.test b/VNFs/DPPD-PROX/helper-scripts/rapid/bare.test
index e3775be9..37d52212 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/bare.test
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/bare.test
@@ -34,13 +34,13 @@ cores = [1]
 [test1]
 test=warmuptest
 flowsize=512
-packetsize=64
+imix=[64]
 warmupspeed=10
 warmuptime=2
 
 [test2]
 test=flowsizetest
-packetsizes=[64,128]
+imixs=[[64],[128]]
 # the number of flows in the list need to be powers of 2, max 2^20
 # Select from following numbers: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576
 flows=[512,1]
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/basicrapid.test b/VNFs/DPPD-PROX/helper-scripts/rapid/basicrapid.test
index 361b0dd7..69a5a309 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/basicrapid.test
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/basicrapid.test
@@ -38,7 +38,7 @@ cores = [1,2]
 [test1]
 test=warmuptest
 flowsize=512
-packetsize=64
+imix=[64]
 warmupspeed=1
 warmuptime=2
 
@@ -49,7 +49,7 @@ test=flowsizetest
 # all combinations must be meeting the same threshold
 # The threshold is expressed in Mpps
 pass_threshold=0.1
-packetsizes=[64,128]
+imixs=[[64], [64,800],[128]]
 # the number of flows in the list need to be powers of 2, max 2^30
 # If not a power of 2, we will use the lowest power of 2 that is larger than
 # the requested number of flows. e.g. 9 will result in 16 flows
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/config_file b/VNFs/DPPD-PROX/helper-scripts/rapid/config_file
index 5e77e31e..c412b616 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/config_file
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/config_file
@@ -3,6 +3,6 @@ cloud_name = openstackL6
 stack_name = rapid
 heat_template = openstack-rapid.yaml
 heat_param = params_rapid.yaml
-keypair_name = prox_key
+keypair_name = rapid_key
 user = centos
 push_gateway = None
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/deploycentostools.sh b/VNFs/DPPD-PROX/helper-scripts/rapid/deploycentostools.sh
index 2f2e6fe0..18b45469 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/deploycentostools.sh
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/deploycentostools.sh
@@ -25,8 +25,12 @@ DPDK_VERSION="19.05"
 PROX_COMMIT="b71a4cfd"
 PROX_CHECKOUT="git checkout ${PROX_COMMIT}"
 ## Next line is overruling the PROX_COMMIT and will replace the version with a very specific patch. Should be commented out
-## 	if you want to use a committed version of PROX
+## 	if you want to use a committed version of PROX with the COMMIT ID specified above
 ##PROX_CHECKOUT="git fetch \"https://gerrit.opnfv.org/gerrit/samplevnf\" refs/changes/75/69475/2 && git checkout FETCH_HEAD"
+##Following line has the commit for testing IMIX
+#PROX_CHECKOUT="git fetch \"https://gerrit.opnfv.org/gerrit/samplevnf\" refs/changes/88/69488/3 && git checkout FETCH_HEAD"
+##Following line has the commit for testing IMIX, IPV6, ... It is the merge of all PROX commits on May 12th
+PROX_CHECKOUT="git fetch \"https://gerrit.opnfv.org/gerrit/samplevnf\" refs/changes/59/69859/7 && git checkout FETCH_HEAD"
 MULTI_BUFFER_LIB_VER="0.52"
 export RTE_SDK="${BUILD_DIR}/dpdk-${DPDK_VERSION}"
 export RTE_TARGET="x86_64-native-linuxapp-gcc"
@@ -95,7 +99,7 @@ function os_cfg()
     # the testing. Once the script connects with ssh, it starts downloading
     # config files and then start prox, etc... The problem is that when the VM
     # boots, check_prox_system_setup.sh will check for some things and
-    # potentially reboot, resulting in loosing the ssh connection again.
+    # potentially reboot, resulting in losing the ssh connection again.
     # To fix this issue, the following lines are disabling ssh access for the 
     # centos user. The script will not be able to connect to the VM till ssh
     # access is restored after a reboot. Restoring ssh is now done by
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/impair.test b/VNFs/DPPD-PROX/helper-scripts/rapid/impair.test
index b8e34568..a6a217de 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/impair.test
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/impair.test
@@ -40,13 +40,13 @@ cores = [1]
 [test1]
 test=warmuptest
 flowsize=1024
-packetsize=64
+imix=[64]
 warmupspeed=10
 warmuptime=2
 
 [test2]
 test=impairtest
-packetsize=64
+imix=[64]
 flowsize=64
 drop_rate_threshold = 0.1
 lat_avg_threshold = 500
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/l2framerate.test b/VNFs/DPPD-PROX/helper-scripts/rapid/l2framerate.test
index 3efbc1a8..fc452f8b 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/l2framerate.test
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/l2framerate.test
@@ -39,5 +39,5 @@ test=fixed_rate
 # The threshold is expressed in Mpps
 pass_threshold=0.1
 startspeed = 10
-packetsizes=[256]
+imixs=[[256]]
 flows=[64]
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/l2zeroloss.test b/VNFs/DPPD-PROX/helper-scripts/rapid/l2zeroloss.test
index bc417c1b..b25038d3 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/l2zeroloss.test
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/l2zeroloss.test
@@ -34,13 +34,13 @@ cores = [1]
 [test1]
 test=warmuptest
 flowsize=512
-packetsize=64
+imix=[64]
 warmupspeed=1
 warmuptime=2
 
 [test2]
 test=flowsizetest
-packetsizes=[64]
+imixs=[[64]]
 # the number of flows in the list need to be powers of 2, max 2^20
 # Select from following numbers: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576
 flows=[512,1]
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/l3framerate.test b/VNFs/DPPD-PROX/helper-scripts/rapid/l3framerate.test
index de67d64a..5b4b914f 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/l3framerate.test
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/l3framerate.test
@@ -34,13 +34,13 @@ cores = [1]
 [test1]
 test=warmuptest
 flowsize=64
-packetsize=64
+imix=[64]
 warmupspeed=1
 warmuptime=2
 
 [test2]
 test=fixed_rate
-packetsizes=[64,128]
+imixs=[[64],[128]]
 # the number of flows in the list need to be powers of 2, max 2^20
 # Select from following numbers: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576
 flows=[1,64]
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
index 6e25e7f4..163c6d9f 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
@@ -321,6 +321,9 @@ class prox_sock(object):
     def set_size(self, cores, task, pkt_size):
         self._send('pkt_size %s %s %s' % (','.join(map(str, cores)), task, pkt_size))
 
+    def set_imix(self, cores, task, imix):
+        self._send('imix %s %s %s' % (','.join(map(str, cores)), task, ','.join(map(str,imix))))
+
     def set_value(self, cores, task, offset, value, length):
         self._send('set value %s %s %s %s %s' % (','.join(map(str, cores)), task, offset, value, length))
 
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py
index d70fd50e..f5155a57 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py
@@ -20,6 +20,7 @@
 import sys
 import time
 from math import ceil
+from statistics import mean
 from past.utils import old_div
 from rapid_log import RapidLog
 from rapid_log import bcolors
@@ -107,13 +108,14 @@ class FlowSizeTest(RapidTest):
     #    writer.writeheader()
         self.gen_machine.start_latency_cores()
         TestPassed = True
-        for size in self.test['packetsizes']:
-            self.gen_machine.set_udp_packet_size(size)
+        for imix in self.test['imixs']:
+            size = mean(imix)
+            self.gen_machine.set_udp_packet_size(imix)
             if self.background_machines:
                 backgroundinfo = '{}Running {} x background traffic not represented in the table{}'.format(bcolors.FLASH,len(self.background_machines),bcolors.ENDC)
             else:
                 backgroundinfo = '{}{}'.format(bcolors.FLASH,bcolors.ENDC)
-            self.set_background_size(self.background_machines, size)
+            self.set_background_size(self.background_machines, imix)
             RapidLog.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
             RapidLog.info('| UDP, {:>5} bytes, different number of flows by randomizing SRC & DST UDP port. {:116.116}|'.format(size, backgroundinfo))
             RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
index 2a5b51c4..cfd0bb11 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
@@ -78,20 +78,37 @@ class RapidGeneratorMachine(RapidMachine):
         speed_per_gen_core = speed / len(self.machine_params['gencores']) 
         self.socket.speed(speed_per_gen_core, self.machine_params['gencores'])
 
-    def set_udp_packet_size(self, frame_size):
+    def set_udp_packet_size(self, imix_frame_sizes):
         # We should check the gen.cfg to make sure we only send UDP packets
-        # Frame size = PROX pkt size + 4 bytes CRC
-        # The set_size function takes the PROX packet size as a parameter
-        self.socket.set_size(self.machine_params['gencores'], 0, frame_size - 4)
-        # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
-        self.socket.set_value(self.machine_params['gencores'], 0, 16, frame_size-18, 2)
-        # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
-        self.socket.set_value(self.machine_params['gencores'], 0, 38, frame_size-38, 2)
+        # If only 1 packet size, still using the 'old' way of setting the 
+        # packet sizes in PROX. Otherwise, using the 'new' way which
+        # automatically sets IP and UDP sizes. We should switch to the new way
+        # eventually for all cases.
+        if len(imix_frame_sizes) == 1:
+            # Frame size = PROX pkt size + 4 bytes CRC
+            # The set_size function takes the PROX packet size as a parameter
+            self.socket.set_size(self.machine_params['gencores'], 0,
+                    imix_frame_sizes[0] - 4)
+            # 18 is the difference between the frame size and IP size =
+            # size of (MAC addresses, ethertype and FCS)
+            self.socket.set_value(self.machine_params['gencores'], 0, 16,
+                    imix_frame_sizes[0] - 18, 2)
+            # 38 is the difference between the frame size and UDP size = 
+            # 18 + size of IP header (=20)
+            self.socket.set_value(self.machine_params['gencores'], 0, 38,
+                    imix_frame_sizes[0] - 38, 2)
+        else:
+            prox_sizes = [frame_size - 4 for frame_size in imix_frame_sizes]
+            self.socket.set_imix(self.machine_params['gencores'], 0,
+                    prox_sizes)
 
     def set_flows(self, number_of_flows):
-        source_port,destination_port = RandomPortBits.get_bitmap(number_of_flows)
-        self.socket.set_random(self.machine_params['gencores'],0,34,source_port,2)
-        self.socket.set_random(self.machine_params['gencores'],0,36,destination_port,2)
+        source_port,destination_port = RandomPortBits.get_bitmap(
+                number_of_flows)
+        self.socket.set_random(self.machine_params['gencores'],0,34,
+                source_port,2)
+        self.socket.set_random(self.machine_params['gencores'],0,36,
+                destination_port,2)
 
     def start_gen_cores(self):
         self.socket.start(self.machine_params['gencores'])
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_impairtest.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_impairtest.py
index 295cb799..eeed2384 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_impairtest.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_impairtest.py
@@ -40,7 +40,8 @@ class ImpairTest(RapidTest):
     #    fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
     #    writer = csv.DictWriter(data_csv_file, fieldnames=fieldnames)
     #    writer.writeheader()
-        size = self.test['packetsize']
+        imix = self.test['imix']
+        size = mean (imix)
         flow_number = self.test['flowsize']
         RapidLog.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
         RapidLog.info("| Generator is sending UDP ("+'{:>5}'.format(flow_number)+" flow) packets ("+ '{:>5}'.format(size) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test    |")
@@ -48,7 +49,7 @@ class ImpairTest(RapidTest):
         RapidLog.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
         RapidLog.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
         attempts = 0
-        self.gen_machine.set_udp_packet_size(size)
+        self.gen_machine.set_udp_packet_size(imix)
         self.gen_machine.set_flows(flow_number)
         self.gen_machine.start_latency_cores()
         speed = self.test['startspeed']
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py
index bebc748e..99e0f697 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py
@@ -59,9 +59,9 @@ class RapidConfigParser(object):
             section = 'test%d'%test_index
             options = testconfig.options(section)
             for option in options:
-                if option in ['packetsizes','flows']:
+                if option in ['imix','imixs','flows']:
                     test[option] = ast.literal_eval(testconfig.get(section, option))
-                    test[option] = [int(i) for i in test[option]]
+#                    test[option] = [int(i) for i in test[option]]
                 elif option in ['maxframespersecondallingress','stepsize']:
                     test[option] = int(testconfig.get(section, option))
                 elif option in ['startspeed','drop_rate_threshold','lat_avg_threshold','lat_perc_threshold','lat_max_threshold','accuracy','maxr','maxz','pass_threshold']:
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py
index 261c8bb1..2466d89b 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py
@@ -52,9 +52,10 @@ class RapidTest(object):
             machine.set_generator_speed(speed)
 
     @staticmethod
-    def set_background_size(background_machines, size):
+    def set_background_size(background_machines, imix):
+        # imixs is a list of packet sizes
         for machine in background_machines:
-            machine.set_udp_packet_size(size)
+            machine.set_udp_packet_size(imix)
 
     @staticmethod
     def start_background_traffic(background_machines):
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_warmuptest.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_warmuptest.py
index 6a9daa73..587ef22a 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_warmuptest.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_warmuptest.py
@@ -35,12 +35,12 @@ class WarmupTest(RapidTest):
     # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
     # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
     # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
-        size = int(self.test['packetsize'])
+        imix = self.test['imix']
         FLOWSIZE = int(self.test['flowsize'])
         WARMUPSPEED = int(self.test['warmupspeed'])
         WARMUPTIME = int(self.test['warmuptime'])
         self.gen_machine.set_generator_speed(WARMUPSPEED)
-        self.gen_machine.set_udp_packet_size(size)
+        self.gen_machine.set_udp_packet_size(imix)
     #    gen_machine['socket'].set_value(gencores,0,56,1,1)
         self.gen_machine.set_flows(FLOWSIZE)
         self.gen_machine.start()
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py
index 5c10b279..67eeea9c 100755
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 ##
 ## Copyright (c) 2010-2020 Intel Corporation
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/secgw.test b/VNFs/DPPD-PROX/helper-scripts/rapid/secgw.test
index 2f09aaf6..fea9b512 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/secgw.test
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/secgw.test
@@ -41,13 +41,13 @@ cores = [1]
 [test1]
 test=warmuptest
 flowsize=512
-packetsize=64
+imix=[64]
 warmupspeed=1
 warmuptime=2
 
 [test2]
 test=flowsizetest
-packetsizes=[64]
+imixs=[[64]]
 # the number of flows in the list need to be powers of 2, max 2^20
 # Select from following numbers: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576
 flows=[512]