Adding vdev support and vfio

Using vfio in stead of igb_uio. DPDK 2020.5 is NOT compiling the igb_uio
driver by default any more.
For the l3 mode, we are now using the new PROX vdev feature. This is
achieved by adding the vdev parameter in the port section of the PROX
config files. One of the advantages is that we can now ping a port
managed by PROX: the requests will be forwarded by PROX to the LINUX
stack, who will reply to the ping. A tap device is being created by PROX
to support this feature.
Installing old version of nasm since latest causes some issues.
If not pushgateway is being used, it should be commented out in the
config_file.
vfio is now loaded by specifying the module in /etc/modules-load.d
There is also an after_boot.sh script that will be run by
check_prox_system_setup.sh, which can be used for further configuration
of the instance after boot.
There is now also a file created /opt/rapid/system_ready_for_rapid by
the check_prox_system_setup.sh script. This file is created once all
configuration is done. The runrapid.py script will wait till this file
is created before executing any tests.

Change-Id: Ic5c41af82642066af42134c3323297f5a06f6f72
Signed-off-by: Luc Provoost <luc.provoost@intel.com>

1 files changed, 19 insertions, 18 deletions

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_impairtest.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_impairtest.py
index 748dcf3e..82067295 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_impairtest.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_impairtest.py
@@ -19,22 +19,21 @@
 
 import sys
 import time
+import requests
 from rapid_log import RapidLog
 from rapid_test import RapidTest
+from statistics import mean
 
 class ImpairTest(RapidTest):
     """
     Class to manage the impair testing
     """
     def __init__(self, test_param, lat_percentile, runtime, pushgateway,
-            environment_file, gen_machine):
-        self.test = test_param
+            environment_file, gen_machine, sut_machine):
+        super().__init__(test_param, runtime, pushgateway, environment_file)
         self.gen_machine = gen_machine
         self.sut_machine = sut_machine
         self.test['lat_percentile'] = lat_percentile
-        self.test['runtime'] = runtime
-        self.test['pushgateway'] = pushgateway
-        self.test['environment_file'] = environment_file
 
     def run(self):
     #    fieldnames = ['Flows','PacketSize','RequestedPPS','GeneratedPPS','SentPPS','ForwardedPPS','ReceivedPPS','AvgLatencyUSEC','MaxLatencyUSEC','Dropped','DropRate']
@@ -47,11 +46,12 @@ class ImpairTest(RapidTest):
         self.gen_machine.set_udp_packet_size(imix)
         flow_number = self.gen_machine.set_flows(flow_number)
         self.gen_machine.start_latency_cores()
-        RapidLog.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
-        RapidLog.info("| Generator is sending UDP ("+'{:>5}'.format(flow_number)+" flow) packets ("+ '{:>5}'.format(size) +" bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test    |")
-        RapidLog.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
-        RapidLog.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
-        RapidLog.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
+        RapidLog.info("+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+")
+        RapidLog.info("| Generator is sending UDP ({:>5} flow) packets ({:>5} bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test                               |".format(flow_number,size))
+        RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
+        RapidLog.info('| Test   | Speed requested  | Gen by core | Sent by NIC | Fwrd by SUT | Rec. by core           | Avg. Lat.|{:.0f} Pcentil| Max. Lat.|   Sent    |  Received |    Lost   | Total Lost|L.Ratio|Time|'.format(self.test['lat_percentile']*100))
+        RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+")
+
         speed = self.test['startspeed']
         self.gen_machine.set_generator_speed(speed)
         while True:
@@ -60,17 +60,18 @@ class ImpairTest(RapidTest):
             sys.stdout.flush()
             time.sleep(1)
             # Get statistics now that the generation is stable and NO ARP messages any more
-            pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg, lat_perc, lat_perc_max, lat_max, abs_dropped, abs_tx_fail, abs_tx, lat_min, lat_used, r, actual_duration = run_iteration(float(self.test['runtime']),flow_number,size,speed)
-            drop_rate = 100.0*abs_dropped/abs_tx
-            if lat_used < 0.95:
-                lat_warning = bcolors.FAIL + ' Potential latency accuracy problem: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
+            pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg, lat_perc, lat_perc_max, lat_max, abs_tx, abs_rx, abs_dropped, abs_tx_fail, drop_rate, lat_min, lat_used, r, actual_duration = self.run_iteration(float(self.test['runtime']),flow_number,size,speed)
+            # Drop rate is expressed in percentage. lat_used is a ratio (0 to 1). The sum of these 2 should be 100%.
+            # If the sum is lower than 95, it means that more than 5% of the latency measurements where dropped for accuracy reasons.
+            if (drop_rate + lat_used * 100) < 95:
+                lat_warning = bcolors.WARNING + ' Latency accuracy issue?: {:>3.0f}%'.format(lat_used*100) +  bcolors.ENDC
             else:
                 lat_warning = ''
-            RapidLog.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |'+lat_warning)
-    #        writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
+            RapidLog.info(self.report_result(attempts,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_perc,lat_perc_max,lat_max,abs_tx,abs_rx,abs_dropped,actual_duration))
+#        writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':self.get_pps(speed,size),'GeneratedPPS':pps_req_tx,'SentPPS':pps_tx,'ForwardedPPS':pps_sut_tx_str,'ReceivedPPS':pps_rx,'AvgLatencyUSEC':lat_avg,'MaxLatencyUSEC':lat_max,'Dropped':abs_dropped,'DropRate':drop_rate})
             if self.test['pushgateway']:
-                URL     = self.test['pushgateway'] + '/metrics/job/' + TestName + '/instance/' + self.test['environment_file'] 
-                DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(flow_number,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
+                URL     = self.test['pushgateway'] + self.test['test'] + '/instance/' + self.test['environment_file'] 
+                DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(flow_number,size+4,self.get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate)
                 HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
                 response = requests.post(url=URL, data=DATA,headers=HEADERS)
                 if (response.status_code != 202) and (response.status_code != 200):