Adding Security gateway testing

Using now .test files describing the test to be executed.
Also desribed in the README file how to deal with SRIOV ports.

Change-Id: I3eb1551d7f205fef3fa1914bc6959c2126d63425
Signed-off-by: Provoost <luc.provoost@intel.com>

1 files changed, 182 insertions, 191 deletions

diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/runrapid.py b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/runrapid.py
index b421c709..7447e160 100755
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/runrapid.py
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/runrapid.py
@@ -31,33 +31,33 @@ from logging import handlers
 from prox_ctrl import prox_ctrl
 import ConfigParser
 
-version="17.09.03"
-stack = "rapidTestEnv" #Default string for stack
+version="17.12.15"
+stack = "rapid" #Default string for stack
+test = "basicrapid" #Default string for stack
 loglevel="DEBUG" # sets log level for writing to file
 runtime=10 # time in seconds for 1 test run
 
 def usage():
-	print("usage: rapid       [--version] [-v]")
+	print("usage: runrapid    [--version] [-v]")
 	print("                   [--stack STACK_NAME]")
+	print("                   [--test TEST_NAME]")
 	print("                   [--runtime TIME_FOR_TEST]")
-	print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL")
+	print("                   [--log DEBUG|INFO|WARNING|ERROR|CRITICAL]")
 	print("                   [-h] [--help]")
 	print("")
-	print("Command-line interface to RAPID")
+	print("Command-line interface to runrapid")
 	print("")
 	print("optional arguments:")
 	print("  -v,  --version           	Show program's version number and exit")
-	print("  --stack STACK_NAME       	Parameters will be read from STACK_NAME.cfg Default is rapidTestEnv.")
+	print("  --stack STACK_NAME       	Parameters will be read from STACK_NAME.env Default is %s."%stack)
+	print("  --test TEST_NAME       	Test cases will be read from TEST_NAME.test Default is %s."%test)
 	print("  --runtime			Specify time in seconds for 1 test run")
 	print("  --log				Specify logging level for log file output, screen output level is hard coded")
 	print("  -h, --help               	Show help message and exit.")
 	print("")
-	print("To delete the rapid stack, type the following command")
-	print("   openstack stack delete --yes --wait rapidTestEnv")
-	print("Note that rapidTestEnv is the default stack name. Replace with STACK_NAME if needed")
 
 try:
-	opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "stack=","runtime=","log="])
+	opts, args = getopt.getopt(sys.argv[1:], "vh", ["version","help", "stack=", "test=","runtime=","log="])
 except getopt.GetoptError as err:
 	print("===========================================")
 	print(str(err))
@@ -77,6 +77,9 @@ for opt, arg in opts:
 	if opt in ("--stack"):
 		stack = arg
 		print ("Using '"+stack+"' as name for the stack")
+	if opt in ("--test"):
+		test = arg
+		print ("Using '"+test+".test' for test case definition")
 	elif opt in ("--runtime"):
 		runtime = arg
 		print ("Runtime: "+ runtime)
@@ -101,8 +104,8 @@ log.setLevel(numeric_level)
 log.propagate = 0
 
 # create a console handler
-# and set its log level to the command-line option
-#
+# and set its log level to the command-line option 
+# 
 console_handler = logging.StreamHandler(sys.stdout)
 console_handler.setLevel(logging.INFO)
 console_handler.setFormatter(screen_formatter)
@@ -110,7 +113,7 @@ console_handler.setFormatter(screen_formatter)
 # create a file handler
 # and set its log level to DEBUG
 #
-log_file = 'RUN' +stack +'.log'
+log_file = 'RUN' +stack+'.'+test+'.log'
 file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10)
 #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5)
 file_handler.setLevel(numeric_level)
@@ -126,7 +129,7 @@ needRoll = os.path.isfile(log_file)
 
 
 # This is a stale log, so roll it
-if needRoll:
+if needRoll:    
     # Add timestamp
     log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime())
 
@@ -150,7 +153,7 @@ def connect_socket(client):
 		if attempts > 20:
 			log.exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
 			raise Exception("Failed to connect to PROX on %s after %d attempts" % (client.ip(), attempts))
-		time.sleep(8)
+		time.sleep(2)
 		log.debug("Trying to connect to PROX (just launched) on %s, attempt: %d" % (client.ip(), attempts))
 	log.info("Connected to PROX on %s" % client.ip())
 	return sock
@@ -167,22 +170,25 @@ def connect_client(client):
 			if attempts > 20:
 				log.exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
 				raise Exception("Failed to connect to VM after %d attempts:\n%s" % (attempts, ex))
-			time.sleep(8)
+			time.sleep(2)
 			log.debug("Trying to connect to VM which was just launched on %s, attempt: %d" % (client.ip(), attempts))
 	log.info("Connected to VM on %s" % client.ip())
 
-def run_iteration(gensock,sutsock,cores,gencores):
-	if sutAdminIP!='none':
-		old_sut_rx, old_sut_tx, old_sut_drop, old_sut_tsc, sut_tsc_hz = sutsock.core_stats([1])
-	old_rx, old_tx, old_drop, old_tsc, tsc_hz = gensock.core_stats(cores)
+def run_iteration(gensock,sutsock,sutstatcores,genstatcores,gencontrolcores):
+	gensock.start(gencontrolcores)
+	time.sleep(1)
+	if sutsock!='none':
+		old_sut_rx, old_sut_tx, old_sut_drop, old_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores)
+	old_rx, old_tx, old_drop, old_tsc, tsc_hz = gensock.core_stats(genstatcores)
 	time.sleep(float(runtime))
+	lat_min, lat_max, lat_avg = gensock.lat_stats([2])
 	# Get statistics after some execution time
-	new_rx, new_tx, new_drop, new_tsc, tsc_hz = gensock.core_stats(cores)
-	if sutAdminIP!='none':
-		new_sut_rx, new_sut_tx, new_sut_drop, new_sut_tsc, sut_tsc_hz = sutsock.core_stats([1])
+	new_rx, new_tx, new_drop, new_tsc, tsc_hz = gensock.core_stats(genstatcores)
+	if sutsock!='none':
+		new_sut_rx, new_sut_tx, new_sut_drop, new_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores)
 	time.sleep(1)
 	#Stop generating
-	gensock.stop(gencores)
+	gensock.stop(gencontrolcores)
 	drop = new_drop-old_drop # drop is all packets dropped by all tasks. This includes packets dropped at the generator task + packets dropped by the nop task. In steady state, this equals to the number of packets received by this VM
 	rx = new_rx - old_rx     # rx is all packets received by the nop task = all packets received in the gen VM
 	tx = new_tx - old_tx     # tx is all generated packets actually accepted by the interface
@@ -190,7 +196,7 @@ def run_iteration(gensock,sutsock,cores,gencores):
 	pps_req_tx = round((tx+drop-rx)*tsc_hz*1.0/(tsc*1000000),3)
 	pps_tx = round(tx*tsc_hz*1.0/(tsc*1000000),3)
 	pps_rx = round(rx*tsc_hz*1.0/(tsc*1000000),3)
-	if sutAdminIP!='none':
+	if sutsock!='none':
 		sut_rx = new_sut_rx - old_sut_rx
 		sut_tx = new_sut_tx - old_sut_tx
 		sut_tsc = new_sut_tsc - old_sut_tsc
@@ -200,9 +206,9 @@ def run_iteration(gensock,sutsock,cores,gencores):
 		pps_sut_tx = 0
 		pps_sut_tx_str = 'NO MEAS.'
 	if (tx == 0):
+        	log.critical("TX = 0. Test interrupted since no packet has been sent.")
 		raise Exception("TX = 0")
-	drop_rate = round(((pps_req_tx-pps_rx) * 100.0)/pps_req_tx,1)
-	return(drop_rate,pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx)
+	return(pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg)
 
 def new_speed(speed,drop_rate):
 	# Following calculates the ratio for the new speed to be applied
@@ -213,96 +219,68 @@ def new_speed(speed,drop_rate):
 	# The second line goes through (p,q) and (100,y100)
 	y0=0.99
 	y100=0.1
-	p=15
-	q=.9
+	p=1
+	q=.99
 	ratio = min((q-y0)/p*drop_rate+y0,(q-y100)/(p-100)*drop_rate+q-p*(q-y100)/(p-100))
 	return (int(speed*ratio*100)+0.5)/100
 
-def run_speedtest():
-        global genclient
-        global sutclient
-        log.info("Starting PROX")
-        speed = 100
-        attempts = 0
-        cores = [1]
-        gencores = [1]
-        cmd = '/root/prox/build/prox -e -t -o cli -f /root/gen.cfg'
-        genclient.fork_cmd(cmd, 'PROX GEN speed Test')
-        gensock = connect_socket(genclient)
-        gensock.reset_stats()
-        if sutAdminIP!='none':
-                cmd = '/root/prox/build/prox -t -o cli -f /root/sut.cfg'
-                sutclient.fork_cmd(cmd, 'PROX SUT speed Test')
-                sutsock = connect_socket(sutclient)
-                sutsock.reset_stats()
-	else:
-		sutsock = 'none'
-        log.info("+-----------------------------------------------------------------------------------------------------------+")
-        log.info("| Generator is sending UDP (1 flow) packets (64 bytes) to SUT. SUT sends packets back                       |")
-        log.info("+--------+-----------------+----------------+----------------+----------------+----------------+------------+")
-        log.info("| Test   | Speed requested | Req to Generate|  Sent by Gen   | Forward by SUT |  Rec. by Gen   | Result     |")
-        log.info("+--------+-----------------+----------------+----------------+----------------+----------------+------------+")
+def get_drop_rate(speed,pps_rx,size):
+	# pps_rx are all the packets that are received by the generator. That is substracted
+	# from the pps that we wanted to send. This is calculated by taking the variable speed
+	# which is the requested percentage of a 10Gb/s link. So we take  10000bps (10Gbps, note
+	# that the speed variable is already expressed in % so we only take 100 and not 10000)
+	# divided by the number of bits in 1 packet. That is 8 bits in a byte times the size of
+	# a frame (=our size + 24 bytes overhead).
+	return (100*(speed * 100 / (8*(size+24)) - pps_rx)/(speed*100.0/(8*(size+24))))
+
+def run_speedtest(gensock,sutsock,sutstatcores,genstatcores,gencores):
+        log.info("+----------------------------------------------------------------------------------------------------------------------------+")
+        log.info("| Generator is sending UDP (1 flow) packets (64 bytes) to SUT. SUT sends packets back                                        |")
+        log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+------------+")
+        log.info("| Test   | Speed requested | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  | Result     |")
+        log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+------------+")
+	speed = 100
+	size=64
+	attempts = 0
         while (speed > 0.1):
                 attempts += 1
                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
                 sys.stdout.flush()
                 # Start generating packets at requested speed (in % of a 10Gb/s link)
                 gensock.speed(speed, gencores)
-                gensock.start(gencores)
                 time.sleep(1)
                 # Get statistics now that the generation is stable and NO ARP messages any more
-		drop_rate,pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx = run_iteration(gensock,sutsock,cores,gencores)
+		pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg = run_iteration(gensock,sutsock,sutstatcores,genstatcores,gencores)
+		drop_rate = get_drop_rate(speed,pps_rx,size)
 	        if ((drop_rate) < 1):
 	                # This will stop the test when number of dropped packets is below a certain percentage
-	                log.info("+--------+-----------------+----------------+----------------+----------------+----------------+------------+")
-	                log.info('|{:>7}'.format(str(attempts))+" | "+ '{:>14}'.format(str(round(speed,2))) + '% | '+ '{:>9}'.format(str(pps_req_tx))+' Mpps | '+ '{:>9}'.format(str(pps_tx)) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9}'.format(str(pps_rx))+" Mpps | SUCCESS    |")
-	                log.info("+--------+-----------------+----------------+----------------+----------------+----------------+------------+")
+			log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+------------+")
+	                log.info('|{:>7}'.format(str(attempts))+" | "+ '{:>14}'.format(str(round(speed,2))) + '% | '+ '{:>9}'.format(str(pps_req_tx))+' Mpps | '+ '{:>9}'.format(str(pps_tx)) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9}'.format(str(pps_rx))+' Mpps | '+ '{:>9}'.format(str(lat_avg))+" us   | SUCCESS    |")
+			log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+------------+")
 	                break
 	        else:
-	                log.info('|{:>7}'.format(str(attempts))+" | "+ '{:>14}'.format(str(round(speed,2))) + '% | '+ '{:>9}'.format(str(pps_req_tx))+' Mpps | '+ '{:>9}'.format(str(pps_tx)) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9}'.format(str(pps_rx))+" Mpps | FAILED     |")
+	                log.info('|{:>7}'.format(str(attempts))+" | "+ '{:>14}'.format(str(round(speed,2))) + '% | '+ '{:>9}'.format(str(pps_req_tx))+' Mpps | '+ '{:>9}'.format(str(pps_tx)) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9}'.format(str(pps_rx))+' Mpps | '+ '{:>9}'.format(str(lat_avg))+" us   | FAILED     |")
 		speed = new_speed(speed,drop_rate)
-        gensock.quit()
-        if sutAdminIP!='none':
-                sutsock.quit()
         time.sleep(2)
 
 
 #	print("")
 
-def run_flowtest():
-	global genclient
-	global sutclient
-	log.info("Starting PROX")
-	speed = 100
-	attempts = 0
-	cores = [1]
-	gencores = [1]
-	cmd = '/root/prox/build/prox -e -t -o cli -f /root/gen.cfg'
-	genclient.fork_cmd(cmd, 'PROX GEN flow Test')
-	gensock = connect_socket(genclient)
-	gensock.reset_stats()
-	if sutAdminIP!='none':
-		cmd = '/root/prox/build/prox -t -o cli -f /root/sut.cfg'
-		sutclient.fork_cmd(cmd, 'PROX SUT flow Test')
-		sutsock = connect_socket(sutclient)
-		sutsock.reset_stats()
-	else:
-		sutsock = 'none'
-	log.info("+----------------------------------------------------------------------------------------------+")
-	log.info("| UDP, 64 bytes, different number of flows by randomizing SRC & DST UDP port                   |")
-	log.info("+--------+-----------------+----------------+----------------+----------------+----------------+")
-	log.info("| Flows  | Speed requested | Req to Generate|  Sent by Gen   | Forward by SUT |  Rec. by Gen   |")
-	log.info("+--------+-----------------+----------------+----------------+----------------+----------------+")
-	cores = [1]
-	gencores = [1]
+def run_flowtest(gensock,sutsock,sutstatcores,genstatcores,gencores):
+	log.info("+---------------------------------------------------------------------------------------------------------------+")
+	log.info("| UDP, 64 bytes, different number of flows by randomizing SRC & DST UDP port                                    |")
+	log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+")
+	log.info("| Flows  | Speed requested | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |")
+	log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+")
 	speed = 100
+	size=64
 	# To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable. 
 	flows={128:['1000000000000XXX','100000000000XXXX'],1024:['10000000000XXXXX','10000000000XXXXX'],8192:['1000000000XXXXXX','100000000XXXXXXX'],65535:['10000000XXXXXXXX','10000000XXXXXXXX'],524280:['1000000XXXXXXXXX','100000XXXXXXXXXX']}
+#	flows={524280:['1000000XXXXXXXXX','100000XXXXXXXXXX']}
 	for flow_number in sorted(flows.iterkeys()):
 		#speed = 100 Commented out: Not starting from 100% since we are trying more flows, so speed will not be higher than the speed achieved in previous loop
-		attempts = 0
 		gensock.reset_stats()
-		if sutAdminIP!='none':
+		if sutsock!='none':
 			sutsock.reset_stats()
 		source_port,destination_port = flows[flow_number]
 		gensock.set_random(gencores,0,34,source_port,2)
@@ -310,59 +288,34 @@ def run_flowtest():
 		while (speed > 0.1):
 			print(str(flow_number)+' flows: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 			sys.stdout.flush()
-			attempts += 1
 			# Start generating packets at requested speed (in % of a 10Gb/s link)
 			gensock.speed(speed, gencores)
-			gensock.start(gencores)
 			time.sleep(1)
 			# Get statistics now that the generation is stable and NO ARP messages any more
-			drop_rate,pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx = run_iteration(gensock,sutsock,cores,gencores)
+			pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg = run_iteration(gensock,sutsock,sutstatcores,genstatcores,gencores)
+			drop_rate = get_drop_rate(speed,pps_rx,size)
 			if ((drop_rate) < 1):
 				# This will stop the test when number of dropped packets is below a certain percentage
-				log.info('|{:>7}'.format(str(flow_number))+" | "+ '{:>14}'.format(str(round(speed,2))) + '% | '+ '{:>9}'.format(str(pps_req_tx))+' Mpps | '+ '{:>9}'.format(str(pps_tx)) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9}'.format(str(pps_rx))+" Mpps |")
-				log.info("+--------+-----------------+----------------+----------------+----------------+----------------+")
+				log.info('|{:>7}'.format(str(flow_number))+" | "+ '{:>14}'.format(str(round(speed,2))) + '% | '+ '{:>9}'.format(str(pps_req_tx))+' Mpps | '+ '{:>9}'.format(str(pps_tx)) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9}'.format(str(pps_rx))+" Mpps |"+ '{:>9}'.format(str(lat_avg))+" us   |")
+				log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+")
 				break
 			speed = new_speed(speed,drop_rate)
-	gensock.quit()
-	if sutAdminIP!='none':
-		sutsock.quit()
 	time.sleep(2)
 #	print("")
 
-def run_sizetest():
-	global genclient
-	global sutclient
-	log.info("Starting PROX")
-	speed = 100
-	attempts = 0
-	cores = [1]
-	gencores = [1]
-	cmd = '/root/prox/build/prox -e -t -o cli -f /root/gen.cfg'
-	genclient.fork_cmd(cmd, 'PROX GEN size Test')
-	gensock = connect_socket(genclient)
-	gensock.reset_stats()
-	if sutAdminIP!='none':
-		cmd = '/root/prox/build/prox -t -o cli -f /root/sut.cfg'
-		sutclient.fork_cmd(cmd, 'PROX SUT size Test')
-		sutsock = connect_socket(sutclient)
-		sutsock.reset_stats()
-	else:
-		sutsock = 'none'
-	log.info("+----------------------------------------------------------------------------------------------+")
-	log.info("| UDP, 1 flow, different packet sizes                                                          |")
-	log.info("+--------+-----------------+----------------+----------------+----------------+----------------+")
-	log.info("| Pktsize| Speed requested | Req to Generate|  Sent by Gen   | Forward by SUT |  Rec. by Gen   |")
-	log.info("+--------+-----------------+----------------+----------------+----------------+----------------+")
-	cores = [1]
-	gencores = [1]
+def run_sizetest(gensock,sutsock,sutstatcores,genstatcores,gencores):
+	log.info("+---------------------------------------------------------------------------------------------------------------+")
+	log.info("| UDP, 1 flow, different packet sizes                                                                           |")
+	log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+")
+	log.info("| Pktsize| Speed requested | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |")
+	log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+")
 	speed = 100
 	# To generate a desired number of flows, PROX will randomize the bits in source and destination ports, as specified by the bit masks in the flows variable. 
-	sizes=[1500,1024,512,256,128,64]
+	sizes=[1400,1024,512,256,128,64]
 	for size in sizes:
 		#speed = 100 Commented out: Not starting from 100% since we are trying smaller packets, so speed will not be higher than the speed achieved in previous loop
-		attempts = 0
 		gensock.reset_stats()
-		if sutAdminIP!='none':
+		if sutsock!='none':
 			sutsock.reset_stats()
 		gensock.set_size(gencores,0,size) # This is setting the frame size
 		gensock.set_value(gencores,0,16,(size-18),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
@@ -371,83 +324,121 @@ def run_sizetest():
 		while (speed > 0.1):
 			print(str(size)+' bytes: Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
 			sys.stdout.flush()
-			attempts += 1
 			# Start generating packets at requested speed (in % of a 10Gb/s link)
 			gensock.speed(speed, gencores)
-			gensock.start(gencores)
 			time.sleep(1)
 			# Get statistics now that the generation is stable and NO ARP messages any more
-			drop_rate,pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx = run_iteration(gensock,sutsock,cores,gencores)
+			pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg = run_iteration(gensock,sutsock,sutstatcores,genstatcores,gencores)
+			drop_rate = get_drop_rate(speed,pps_rx,size)
 			if ((drop_rate) < 1):
 				# This will stop the test when number of dropped packets is below a certain percentage
-				log.info('|{:>7}'.format(str(size))+" | "+ '{:>14}'.format(str(round(speed,2))) + '% | '+ '{:>9}'.format(str(pps_req_tx))+' Mpps | '+ '{:>9}'.format(str(pps_tx)) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9}'.format(str(pps_rx))+" Mpps |")
-				log.info("+--------+-----------------+----------------+----------------+----------------+----------------+")
+				log.info('|{:>7}'.format(str(size))+" | "+ '{:>14}'.format(str(round(speed,2))) + '% | '+ '{:>9}'.format(str(pps_req_tx))+' Mpps | '+ '{:>9}'.format(str(pps_tx)) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9}'.format(str(pps_rx))+" Mpps |"+ '{:>10}'.format(str(lat_avg))+" us   |")
+				log.info("+--------+-----------------+----------------+----------------+----------------+----------------+----------------+")
 				break
 			speed = new_speed(speed,drop_rate)
-	gensock.quit()
-	if sutAdminIP!='none':
-		sutsock.quit()
 	time.sleep(2)
 #========================================================================
+
+def init_test():
+	global sutstatcores
+	global genstatcores
+	global genrxcores
+	global gencontrolcores
+	sutstatcores = [1]
+	genstatcores = [1,2]
+	genrxcores = [2]
+	gencontrolcores = [1]
+# Running at low speed to make sure the ARP messages can get through.
+# If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
+# Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
+# PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
+	sock[0].speed(0.01, gencontrolcores)
+	sock[0].start(genstatcores)
+	time.sleep(2)
+	sock[0].stop(gencontrolcores)
+	sock[1].start([1])
+
+vmDPIP =[]
+vmAdminIP =[]
+vmDPmac =[]
+hexDPIP =[]
+config_file =[]
+script_control =[]
+
+testconfig = ConfigParser.RawConfigParser()
+testconfig.read(test+'.test')
+required_number_of_VMs = testconfig.get('DEFAULT', 'total_number_of_vms')
 config = ConfigParser.RawConfigParser()
-config.read(stack+'.cfg')
-
-genAdminIP = config.get('Generator', 'admin_ip')
-genDPmac = config.get('Generator', 'dp_mac')
-genDPIP = config.get('Generator', 'dp_ip')
-sutAdminIP = config.get('SUT', 'admin_ip')
-sutDPmac = config.get('SUT', 'dp_mac')
-sutDPIP = config.get('SUT', 'dp_ip')
+config.read(stack+'.env')
 key = config.get('OpenStack', 'key')
-ip = genDPIP.split('.')
-hexgenDPIP=hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2)
-ip = sutDPIP.split('.')
-hexsutDPIP=hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2)
-with open("parameters.lua", "w") as f:
-        f.write('gen_hex_ip="'+hexgenDPIP+'"\n')
-        f.write('sut_hex_ip="'+hexsutDPIP+'"\n')
-        f.write('gen_ip="'+genDPIP+'"\n')
-        f.write('sut_ip="'+sutDPIP+'"\n')
-        f.close
-
+total_number_of_VMs = config.get('rapid', 'total_number_of_VMs')
+if int(required_number_of_VMs) > int(total_number_of_VMs):
+	log.exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_VMs,total_number_of_VMs))
+	raise Exception("Not enough VMs for this test: %s needed and only %s available" % (required_number_of_VMs,total_number_of_VMs))
+for vm in range(1, int(total_number_of_VMs)+1):
+	vmAdminIP.append(config.get('VM%d'%vm, 'admin_ip'))
+	vmDPmac.append(config.get('VM%d'%vm, 'dp_mac'))
+	vmDPIP.append(config.get('VM%d'%vm, 'dp_ip'))
+	ip = vmDPIP[-1].split('.')
+	hexDPIP.append(hex(int(ip[0]))[2:].zfill(2) + ' ' + hex(int(ip[1]))[2:].zfill(2) + ' ' + hex(int(ip[2]))[2:].zfill(2) + ' ' + hex(int(ip[3]))[2:].zfill(2))
+for vm in range(1, int(required_number_of_VMs)+1):
+	config_file.append(testconfig.get('VM%d'%vm, 'config_file'))
+	script_control.append(testconfig.get('VM%d'%vm, 'script_control'))
+	with open("parameters%d.lua"%vm, "w") as f:
+		f.write('name="%s"\n'% testconfig.get('VM%d'%vm, 'name'))
+		f.write('local_ip="%s"\n'% vmDPIP[vm-1])
+		f.write('local_hex_ip="%s"\n'% hexDPIP[vm-1])
+		gwVM = testconfig.get('VM%d'%vm, 'gw_vm')
+		if gwVM <> 'not_used':
+			gwVMindex = int(gwVM)-1
+			f.write('gw_ip="%s"\n'% vmDPIP[gwVMindex])
+			f.write('gw_hex_ip="%s"\n'% hexDPIP[gwVMindex])
+		destVM = testconfig.get('VM%d'%vm, 'dest_vm')
+		if destVM <> 'not_used':
+			destVMindex = int(destVM)-1
+			f.write('dest_ip="%s"\n'% vmDPIP[destVMindex])
+			f.write('dest_hex_ip="%s"\n'% hexDPIP[destVMindex])
+	f.close
 #####################################################################################
-genclient = prox_ctrl(genAdminIP, key+'.pem','root')
-connect_client(genclient)
-genclient.scp_put('./gen.cfg', '/root/gen.cfg')
-genclient.scp_put('./parameters.lua', '/root/parameters.lua')
-# Creating script to bind the right network interface to the poll mode driver
-with open("devbind.sh") as f:
-    newText=f.read().replace('MACADDRESS', genDPmac)
-with open("gendevbind.sh", "w") as f:
-    f.write(newText)
-st = os.stat('gendevbind.sh')
-os.chmod('gendevbind.sh', st.st_mode | stat.S_IEXEC)
-genclient.scp_put('./gendevbind.sh', '/root/gendevbind.sh')
-cmd = '/root/gendevbind.sh'
-genclient.run_cmd(cmd)
-log.info("Generator Config files copied & running devbind.sh")
+client =[]
+sock =[]
 
-#####################################################################################
-if sutAdminIP!='none':
-	sutclient = prox_ctrl(sutAdminIP, key+'.pem','root')
-	connect_client(sutclient)
-	sutclient.scp_put('./sut.cfg', '/root/sut.cfg')
-	sutclient.scp_put('./parameters.lua', '/root/parameters.lua')
+for vm in range(0, int(required_number_of_VMs)):
+	client.append(prox_ctrl(vmAdminIP[vm], key+'.pem','root'))
+	connect_client(client[-1])
 # Creating script to bind the right network interface to the poll mode driver
+	devbindfile = "devbindvm%d.sh"%(vm+1)
 	with open("devbind.sh") as f:
-	    newText=f.read().replace('MACADDRESS', sutDPmac)
-	with open("sutdevbind.sh", "w") as f:
-	    f.write(newText)
-	st = os.stat('sutdevbind.sh')
-	os.chmod('sutdevbind.sh', st.st_mode | stat.S_IEXEC)
-	sutclient.scp_put('./sutdevbind.sh', '/root/sutdevbind.sh')
-	cmd = '/root/sutdevbind.sh'
-	sutclient.run_cmd(cmd)
-	log.info("SUT Config files copied & running devbind.sh")
-run_speedtest()
-run_flowtest()
-run_sizetest()
-#####################################################################################
-genclient.close()
-if sutAdminIP!='none':
-	sutclient.close()
+		newText=f.read().replace('MACADDRESS', vmDPmac[vm])
+		with open(devbindfile, "w") as f:
+			f.write(newText)
+	st = os.stat(devbindfile)
+	os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
+	client[-1].scp_put('./%s'%devbindfile, '/root/devbind.sh')
+	cmd = '/root/devbind.sh'
+	client[-1].run_cmd(cmd)
+	log.info("devbind.sh running on VM%d"%(vm+1))
+	client[-1].scp_put('./%s'%config_file[vm], '/root/%s'%config_file[vm])
+	client[-1].scp_put('./parameters%d.lua'%(vm+1), '/root/parameters.lua')
+	log.info("Starting PROX on VM%d"%(vm+1))
+	if script_control[vm] == 'true':
+		cmd = '/root/prox/build/prox -e -t -o cli -f /root/%s'%config_file[vm]
+	else:
+		cmd = '/root/prox/build/prox -t -o cli -f /root/%s'%config_file[vm]
+	client[-1].fork_cmd(cmd, 'PROX Testing on VM%d'%(vm+1))
+	sock.append(connect_socket(client[-1]))
+
+init_code = testconfig.get('DEFAULT', 'init_code')
+eval(init_code)
+####################################################
+# Run test cases
+# Best to run the flow test at the end since otherwise the tests coming after thatmight be influenced by the big number of entries in the switch flow tables
+####################################################
+number_of_tests = testconfig.get('DEFAULT', 'number_of_tests')
+for vm in range(1, int(number_of_tests)+1):
+	cmd=testconfig.get('test%d'%vm,'cmd')
+	eval(cmd)
+####################################################
+for vm in range(0, int(required_number_of_VMs)):
+	sock[vm].quit()
+	client[vm].close()