Code rewrite and Python3 support

The code has been rewritten in multiple classes and files for better
readability and maintainability. You should now also use Python3 to run
the scripts. Some of the variables in the config files have moved to
other sections. The pushgateway parameter moved to the [varia] section
of the rapid environment file.

Change-Id: Ic61a1bc3baa39447d42b21dfcc0f16c28fcdfa6e
Signed-off-by: Luc Provoost <luc.provoost@intel.com>

1 files changed, 108 insertions, 95 deletions

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
index ba21913c..82faa789 100644
--- a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
+++ b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py
@@ -1,5 +1,5 @@
 ##
-## Copyright (c) 2010-2019 Intel Corporation
+## Copyright (c) 2010-2020 Intel Corporation
 ##
 ## Licensed under the Apache License, Version 2.0 (the "License");
 ## you may not use this file except in compliance with the License.
@@ -15,10 +15,17 @@
 ##
 
 from __future__ import print_function
+from __future__ import division
 
+from builtins import map
+from builtins import range
+from past.utils import old_div
+from builtins import object
 import os
+import time
 import subprocess
 import socket
+from rapid_log import RapidLog 
 
 class prox_ctrl(object):
     def __init__(self, ip, key=None, user=None):
@@ -28,20 +35,55 @@ class prox_ctrl(object):
         self._children = []
         self._proxsock = []
 
+    def __del__(self):
+        self.close()
+
     def ip(self):
         return self._ip
 
-    def connect(self):
+    def test_connect(self):
         """Simply try to run 'true' over ssh on remote system.
         On failure, raise RuntimeWarning exception when possibly worth
         retrying, and raise RuntimeError exception otherwise.
         """
         return self.run_cmd('true', True)
+    def connect(self):
+        attempts = 1
+        RapidLog.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (self._ip, attempts))
+        while True:
+            try:
+                self.test_connect()
+                break
+            except RuntimeWarning as ex:
+                attempts += 1
+                if attempts > 20:
+                    RapidLog.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
+                    raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
+                time.sleep(2)
+                RapidLog.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (self._ip, attempts))
+        RapidLog.debug("Connected to VM on %s" % self._ip)
+
+    def connect_socket(self):
+        attempts = 1
+        RapidLog.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (self._ip, attempts))
+        sock = None
+        while True:
+            sock = self.prox_sock()
+            if sock is not None:
+                break
+            attempts += 1
+            if attempts > 20:
+                RapidLog.exception("Failed to connect to PROX on %s after %d attempts" % (self._ip, attempts))
+                raise Exception("Failed to connect to PROX on %s after %d attempts" % (self._ip, attempts))
+            time.sleep(2)
+            RapidLog.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (self._ip, attempts))
+        RapidLog.info("Connected to PROX on %s" % self._ip)
+        return sock
 
     def close(self):
         """Must be called before program termination."""
-#        for prox in self._proxsock:
-#            prox.quit()
+        for sock in self._proxsock:
+            sock.quit()
         children = len(self._children)
         if children == 0:
             return
@@ -160,6 +202,9 @@ class prox_sock(object):
         self._sock = sock
         self._rcvd = b''
 
+    def __del__(self):
+        self.quit()
+
     def quit(self):
         if self._sock is not None:
             self._send('quit')
@@ -174,81 +219,50 @@ class prox_sock(object):
 
     def speed(self, speed, cores, tasks=[0]):
         for core in cores:
-        	for task in tasks:
-			self._send('speed %s %s %s' % (core, task, speed))
+            for task in tasks:
+                self._send('speed %s %s %s' % (core, task, speed))
 
     def reset_stats(self):
         self._send('reset stats')
 
     def lat_stats(self, cores, tasks=[0]):
         min_lat = 999999999
-	max_lat = avg_lat = 0
-	number_tasks_returning_stats = 0
-	buckets = [0] * 128
+        max_lat = avg_lat = 0
+        number_tasks_returning_stats = 0
+        buckets = [0] * 128
         self._send('lat all stats %s %s' % (','.join(map(str, cores)), ','.join(map(str, tasks))))
         for core in cores:
-	    	for task in tasks:
-	        	stats = self._recv().split(',')
-			if 'is not measuring' in stats[0]:
-				continue
-			if stats[0].startswith('error'):
-				log.critical("lat stats error: unexpected reply from PROX (potential incompatibility between scripts and PROX)")
-				raise Exception("lat stats error")
-			number_tasks_returning_stats += 1
-			min_lat = min(int(stats[0]),min_lat)
-			max_lat = max(int(stats[1]),max_lat)
-			avg_lat += int(stats[2])
-			#min_since begin = int(stats[3])
-			#max_since_begin = int(stats[4])
-			tsc = int(stats[5]) # Taking the last tsc as the timestamp since PROX will return the same tsc for each core/task combination 
-			hz = int(stats[6])
-			#coreid = int(stats[7])
-			#taskid = int(stats[8])
-	        	stats = self._recv().split(':')
-			if stats[0].startswith('error'):
-				log.critical("lat stats error: unexpected lat bucket reply (potential incompatibility between scripts and PROX)")
-				raise Exception("lat bucket reply error")
-			buckets[0] = int(stats[1])
-			for i in range(1, 128):
-	        		stats = self._recv().split(':')
-				buckets[i] = int(stats[1])
-        avg_lat = avg_lat/number_tasks_returning_stats
+            for task in tasks:
+                stats = self._recv().split(',')
+            if 'is not measuring' in stats[0]:
+                continue
+            if stats[0].startswith('error'):
+                RapidLog.critical("lat stats error: unexpected reply from PROX (potential incompatibility between scripts and PROX)")
+                raise Exception("lat stats error")
+            number_tasks_returning_stats += 1
+            min_lat = min(int(stats[0]),min_lat)
+            max_lat = max(int(stats[1]),max_lat)
+            avg_lat += int(stats[2])
+            #min_since begin = int(stats[3])
+            #max_since_begin = int(stats[4])
+            tsc = int(stats[5]) # Taking the last tsc as the timestamp since PROX will return the same tsc for each core/task combination 
+            hz = int(stats[6])
+            #coreid = int(stats[7])
+            #taskid = int(stats[8])
+            stats = self._recv().split(':')
+            if stats[0].startswith('error'):
+                RapidLog.critical("lat stats error: unexpected lat bucket reply (potential incompatibility between scripts and PROX)")
+                raise Exception("lat bucket reply error")
+            buckets[0] = int(stats[1])
+            for i in range(1, 128):
+                stats = self._recv().split(':')
+                buckets[i] = int(stats[1])
+        avg_lat = old_div(avg_lat,number_tasks_returning_stats)
         self._send('stats latency(0).used')
         used = float(self._recv())
         self._send('stats latency(0).total')
         total = float(self._recv())
-        return min_lat, max_lat, avg_lat, (used/total), tsc, hz, buckets
-
-    def old_lat_stats(self, cores, tasks=[0]):
-        min_lat = 999999999
-	max_lat = avg_lat = 0
-	number_tasks_returning_stats = 0
-	buckets = [0] * 128
-        self._send('lat stats %s %s' % (','.join(map(str, cores)), ','.join(map(str, tasks))))
-        for core in cores:
-	    	for task in tasks:
-	        	stats = self._recv().split(',')
-			if stats[0].startswith('error'):
-				if stats[0].startswith('error: invalid syntax'):
-		                	log.critical("lat stats error: unexpected invalid syntax (potential incompatibility between scripts and PROX)")
-			                raise Exception("lat stats error")
-				continue
-			number_tasks_returning_stats += 1
-			min_lat = min(int(stats[0]),min_lat)
-			max_lat = max(int(stats[1]),max_lat)
-			avg_lat += int(stats[2])
-			#min_since begin = int(stats[3])
-			#max_since_begin = int(stats[4])
-			tsc = int(stats[5])
-			hz = int(stats[6])
-			#coreid = int(stats[7])
-			#taskid = int(stats[8])
-        avg_lat = avg_lat/number_tasks_returning_stats
-        self._send('stats latency(0).used')
-        used = float(self._recv())
-        self._send('stats latency(0).total')
-        total = float(self._recv())
-        return min_lat, max_lat, avg_lat, (used/total), tsc, hz, buckets
+        return min_lat, max_lat, avg_lat, (old_div(used,total)), tsc, hz, buckets
 
     def irq_stats(self, core, bucket, task=0):
         self._send('stats task.core(%s).task(%s).irq(%s)' % (core, task, bucket))
@@ -259,45 +273,45 @@ class prox_sock(object):
         rx = tx = drop = tsc = hz = 0
         self._send('show irq buckets %s %s' % (core,task))
         buckets = self._recv().split(';')
-	buckets = buckets[:-1]
+        buckets = buckets[:-1]
         return buckets
 
     def core_stats(self, cores, tasks=[0]):
         rx = tx = drop = tsc = hz = rx_non_dp = tx_non_dp = tx_fail = 0
         self._send('dp core stats %s %s' % (','.join(map(str, cores)), ','.join(map(str, tasks))))
         for core in cores:
-        	for task in tasks:
-			stats = self._recv().split(',')
-			if stats[0].startswith('error'):  
-				if stats[0].startswith('error: invalid syntax'):
-					log.critical("dp core stats error: unexpected invalid syntax (potential incompatibility between scripts and PROX)")
-					raise Exception("dp core stats error")
-				continue
-			rx += int(stats[0])
-			tx += int(stats[1])
-			rx_non_dp += int(stats[2])
-			tx_non_dp += int(stats[3])
-			drop += int(stats[4])
-			tx_fail += int(stats[5])
-			tsc = int(stats[6])
-			hz = int(stats[7])
+            for task in tasks:
+                stats = self._recv().split(',')
+                if stats[0].startswith('error'):  
+                    if stats[0].startswith('error: invalid syntax'):
+                        RapidLog.critical("dp core stats error: unexpected invalid syntax (potential incompatibility between scripts and PROX)")
+                        raise Exception("dp core stats error")
+                    continue
+                rx += int(stats[0])
+                tx += int(stats[1])
+                rx_non_dp += int(stats[2])
+                tx_non_dp += int(stats[3])
+                drop += int(stats[4])
+                tx_fail += int(stats[5])
+                tsc = int(stats[6])
+                hz = int(stats[7])
         return rx, rx_non_dp, tx, tx_non_dp, drop, tx_fail, tsc, hz
 
     def multi_port_stats(self, ports=[0]):
         rx = tx = port_id = tsc = no_mbufs = errors = 0
         self._send('multi port stats %s' % (','.join(map(str, ports))))
-	result = self._recv().split(';')
-	if result[0].startswith('error'):  
-		log.critical("multi port stats error: unexpected invalid syntax (potential incompatibility between scripts and PROX)")
-		raise Exception("multi port stats error")
+        result = self._recv().split(';')
+        if result[0].startswith('error'):  
+            RapidLog.critical("multi port stats error: unexpected invalid syntax (potential incompatibility between scripts and PROX)")
+            raise Exception("multi port stats error")
         for statistics in result:
-		stats = statistics.split(',')
-		port_id = int(stats[0])
-		rx += int(stats[1])
-		tx += int(stats[2])
-		no_mbufs += int(stats[3])
-		errors += int(stats[4])
-		tsc = int(stats[5])
+            stats = statistics.split(',')
+            port_id = int(stats[0])
+            rx += int(stats[1])
+            tx += int(stats[2])
+            no_mbufs += int(stats[3])
+            errors += int(stats[4])
+            tsc = int(stats[5])
         return rx, tx, no_mbufs, errors, tsc
 
     def set_random(self, cores, task, offset, mask, length):
@@ -326,4 +340,3 @@ class prox_sock(object):
         rsp = self._rcvd[:pos]
         self._rcvd = self._rcvd[pos+1:]
         return rsp.decode()
-