From a9f2cbbc2d12d6c767341e675adc5921c39c21a9 Mon Sep 17 00:00:00 2001 From: Luc Provoost Date: Mon, 23 Mar 2020 06:53:39 -0400 Subject: Improved variable init and bug fixes Variables were initiales based on the file name of the PROX configuration files. These decisions are now based on the content of these files and other variables, rather than just the name of a file. This commit also fixes some bugs: irq test was not executing any more, some issues with the push gateway reporting. Change-Id: I39308effad3c80cf52d96e4d85d92d4b5f0be9a5 Signed-off-by: Luc Provoost --- VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py | 103 ++++++++++-------------- 1 file changed, 41 insertions(+), 62 deletions(-) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py index 62164c4b..5520d139 100755 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py @@ -37,7 +37,7 @@ import requests from math import ceil inf = float("inf") -version="20.01.10" +version="20.03.23" env = "rapid.env" #Default string for environment test_file = "basicrapid.test" #Default string for test machine_map_file = "machine.map" #Default string for machine map file @@ -45,7 +45,8 @@ loglevel="DEBUG" # sets log level for writing to file screenloglevel="INFO" # sets log level for writing to screen runtime=10 # time in seconds for 1 test run configonly = False # IF True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing -rundir = "/home/centos" # Directory where to find the tools in the machines running PROX +#rundir = "/home/centos" # Directory where to find the tools in the machines running PROX +rundir = "~" # Directory where to find the tools in the machines running PROX def usage(): print("usage: runrapid [--version] [-v]") @@ -231,7 +232,7 @@ def report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat else: pps_sut_tx_str = '{:>7.3f} Mpps |'.format(pps_sut_tx) if pps_rx is None: - pps_rx_str = '{0: >24|}'.format('NA ') + pps_rx_str = '{0: >25}'.format('NA |') else: pps_rx_str = bcolors.OKBLUE + '{:>4.1f} Gb/s |{:7.3f} Mpps {}|'.format(get_speed(pps_rx,size),pps_rx,bcolors.ENDC) if tot_drop is None: @@ -241,14 +242,14 @@ def report_result(flow_number,size,speed,pps_req_tx,pps_tx,pps_sut_tx,pps_rx,lat if lat_perc is None: lat_perc_str = ' |{:^10.10}|'.format('NA') elif lat_perc_max == True: - lat_perc_str = ' |>{}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC) + lat_perc_str = '|>{}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC) else: - lat_perc_str = ' | {}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC) + lat_perc_str = '| {}{:>5.0f} us{} |'.format(lat_perc_prefix,float(lat_perc), bcolors.ENDC) if elapsed_time is None: elapsed_time_str = ' NA |' else: elapsed_time_str = '{:>3.0f} |'.format(elapsed_time) - return(flow_number_str + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ pps_req_tx_str + pps_tx_str + bcolors.ENDC + pps_sut_tx_str + pps_rx_str +lat_avg_prefix+ ' {:>5.0f}'.format(lat_avg)+' us'+lat_perc_str+lat_max_prefix+'{:>6.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | {:>9.0f}'.format(rx) + ' | '+ abs_drop_rate_prefix+ '{:>9.0f}'.format(tx-rx) + tot_drop_str +drop_rate_prefix+ '{:>5.2f}'.format(float(tx-rx)/tx) +bcolors.ENDC+' |' + elapsed_time_str) + return(flow_number_str + '{:>5.1f}'.format(speed) + '% '+speed_prefix +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps|'+ pps_req_tx_str + pps_tx_str + bcolors.ENDC + pps_sut_tx_str + pps_rx_str +lat_avg_prefix+ ' {:>6.0f}'.format(lat_avg)+' us'+lat_perc_str+lat_max_prefix+'{:>6.0f}'.format(lat_max)+' us | ' + '{:>9.0f}'.format(tx) + ' | {:>9.0f}'.format(rx) + ' | '+ abs_drop_rate_prefix+ '{:>9.0f}'.format(tx-rx) + tot_drop_str +drop_rate_prefix+ '{:>5.2f}'.format(float(tx-rx)/tx) +bcolors.ENDC+' |' + elapsed_time_str) def run_iteration(gensock, sutsock, requested_duration,flow_number,size,speed): r = 0; @@ -260,11 +261,9 @@ def run_iteration(gensock, sutsock, requested_duration,flow_number,size,speed): t1_dp_rx = t1_rx - t1_non_dp_rx t1_dp_tx = t1_tx - t1_non_dp_tx gensock.start(gencores) - time.sleep(2) ## Needs to be 2 seconds since this the time that PROX uses to refresh the stats. Note that this can be changed in PROX!! Don't do it. + time.sleep(2) ## Needs to be 2 seconds since this 1 sec is the time that PROX uses to refresh the stats. Note that this can be changed in PROX!! Don't do it. if sutsock!='none': t2_sut_rx, t2_sut_non_dp_rx, t2_sut_tx, t2_sut_non_dp_tx, t2_sut_drop, t2_sut_tx_fail, t2_sut_tsc, sut_tsc_hz = sutsock.core_stats(sutstatcores,tasks) - ##t2_sut_rx = t2_sut_rx - t2_sut_non_dp_rx - ##t2_sut_tx = t2_sut_tx - t2_sut_non_dp_tx t2_rx, t2_non_dp_rx, t2_tx, t2_non_dp_tx, t2_drop, t2_tx_fail, t2_tsc, tsc_hz = gensock.core_stats(genstatcores,gentasks) tx = t2_tx - t1_tx dp_tx = tx - (t2_non_dp_tx - t1_non_dp_tx ) @@ -578,8 +577,8 @@ def run_flow_size_test(gensock,sutsock): enddrop_rate = drop_rate endabs_tx = abs_tx endabs_rx = abs_rx - if lat_warning or gen_warning or retry_warning: - endwarning = '| | {:177.177} |'.format(retry_warning + lat_warning + gen_warning) + if lat_warning or retry_warning: + endwarning = '| | {:177.177} |'.format(retry_warning + lat_warning) success = True speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold @@ -655,9 +654,16 @@ def run_flow_size_test(gensock,sutsock): writer.writerow({'Flows':flow_number,'PacketSize':(size+4),'RequestedPPS':get_pps(endspeed,size),'GeneratedPPS':endpps_req_tx,'SentPPS':endpps_tx,'ForwardedPPS':endpps_sut_tx,'ReceivedPPS':endpps_rx,'AvgLatencyUSEC':endlat_avg,'MaxLatencyUSEC':endlat_max,'Sent':endabs_tx,'Received':endabs_rx,'Lost':endabs_dropped,'LostTotal':endabs_dropped}) if PushGateway: URL = PushGateway + '/metrics/job/' + TestName + '/instance/' + env - DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,endabs_Dropped,endabs_dropped) + if endabs_dropped == None: + ead = 0 + else: + ead = endabs_dropped + DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nSent {}\nReceived {}\nLost {}\nLostTotal {}\n'.format(flow_number,size+4,get_pps(endspeed,size),endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_max,endabs_tx,endabs_rx,ead,ead) HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'} response = requests.post(url=URL, data=DATA,headers=HEADERS) + if (response.status_code != 202) and (response.status_code != 200): + log.info('Cannot send metrics to {}'.format(URL)) + log.info(DATA) else: log.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0") gensock.stop(latcores) @@ -711,6 +717,9 @@ def run_core_stats(socks): DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNonDPReceived {}\nNonDPSent {}\nDelta {}\nNonDPDelta {}\nDropped {}\n'.format(i,duration,rx,tx,non_dp_rx,non_dp_tx,tx-rx,non_dp_tx-non_dp_rx,tot_drop[i]) HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'} response = requests.post(url=URL, data=DATA,headers=HEADERS) + if (response.status_code != 202) and (response.status_code != 200): + log.info('Cannot send metrics to {}'.format(URL)) + log.info(DATA) if sockets_to_go == 0: duration = duration - 1 sockets_to_go = len (socks) @@ -760,6 +769,9 @@ def run_port_stats(socks): DATA = 'PROXID {}\nTime {}\n Received {}\nSent {}\nNoMbufs {}\niErrMiss {}\n'.format(i,duration,rx,tx,no_mbufs,errors) HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'} response = requests.post(url=URL, data=DATA,headers=HEADERS) + if (response.status_code != 202) and (response.status_code != 200): + log.info('Cannot send metrics to {}'.format(URL)) + log.info(DATA) if sockets_to_go == 0: duration = duration - 1 sockets_to_go = len (socks) @@ -843,6 +855,9 @@ def run_impairtest(gensock,sutsock): DATA = 'Flows {}\nPacketSize {}\nRequestedPPS {}\nGeneratedPPS {}\nSentPPS {}\nForwardedPPS {}\nReceivedPPS {}\nAvgLatencyUSEC {}\nMaxLatencyUSEC {}\nDropped {}\nDropRate {}\n'.format(FLOWSIZE,size+4,get_pps(speed,size),pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max,abs_dropped,drop_rate) HEADERS = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'} response = requests.post(url=URL, data=DATA,headers=HEADERS) + if (response.status_code != 202) and (response.status_code != 200): + log.info('Cannot send metrics to {}'.format(URL)) + log.info(DATA) def run_warmuptest(gensock): # Running at low speed to make sure the ARP messages can get through. @@ -982,7 +997,6 @@ vmDPPCIDEV =[] config_file =[] prox_socket =[] prox_launch_exit =[] -auto_start =[] mach_type =[] sock_type =[] monitor =[] @@ -1028,6 +1042,7 @@ for vm in range(1, int(total_number_of_machines)+1): if (vim_type == "kubernetes"): vmDPPCIDEV.append(config.get('M%d'%vm, 'dp_pci_dev')) machine_index = [] +atexit.register(exit_handler) for vm in range(1, int(required_number_of_test_machines)+1): machine_index.append(int(machine_map.get('TestM%d'%vm, 'machine_index'))-1) prox_socket.append(testconfig.getboolean('TestM%d'%vm, 'prox_socket')) @@ -1062,12 +1077,12 @@ for vm in range(1, int(required_number_of_test_machines)+1): monitor.append(testconfig.getboolean('TestM%d'%vm, 'monitor')) else: monitor.append(True) - if re.match('(l2){0,1}gen(_bare){0,1}.*\.cfg',config_file[-1]): + if testconfig.has_option('TestM%d'%vm, 'gencores'): + # This must be a generator VM gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores')) latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores')) genstatcores = gencores + latcores gentasks = tasks_for_this_cfg - auto_start.append(False) mach_type.append('gen') f.write('gencores="%s"\n'% ','.join(map(str, gencores))) f.write('latcores="%s"\n'% ','.join(map(str, latcores))) @@ -1075,6 +1090,10 @@ for vm in range(1, int(required_number_of_test_machines)+1): f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]]) f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]]) f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' ')) + if testconfig.has_option('TestM%d'%vm, 'gw_vm'): + gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1 + f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]]) + f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]]) if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'): BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp')) else: @@ -1085,53 +1104,17 @@ for vm in range(1, int(required_number_of_test_machines)+1): else: heartbeat = 60 f.write('heartbeat="%s"\n'% heartbeat) - elif re.match('(l2){0,1}gen_gw.*\.cfg',config_file[-1]): - if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'): - BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp')) - else: - BUCKET_SIZE_EXP = 11 - gencores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'gencores')) - latcores = ast.literal_eval(testconfig.get('TestM%d'%vm, 'latcores')) - genstatcores = gencores + latcores - gentasks = tasks_for_this_cfg - auto_start.append(False) - mach_type.append('gen') - f.write('gencores="%s"\n'% ','.join(map(str, gencores))) - f.write('latcores="%s"\n'% ','.join(map(str, latcores))) - gwVMindex = int(testconfig.get('TestM%d'%vm, 'gw_vm')) -1 - f.write('gw_ip="%s"\n'% vmDPIP[machine_index[gwVMindex]]) - f.write('gw_hex_ip="%s"\n'% hexDPIP[machine_index[gwVMindex]]) - destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1 - f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]]) - f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]]) - f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' ')) - if testconfig.has_option('TestM%d'%vm, 'bucket_size_exp'): - BUCKET_SIZE_EXP = int(testconfig.get('TestM%d'%vm, 'bucket_size_exp')) - else: - BUCKET_SIZE_EXP = 11 - f.write('bucket_size_exp="%s"\n'% BUCKET_SIZE_EXP) - if testconfig.has_option('TestM%d'%vm, 'heartbeat'): - heartbeat = int(testconfig.get('TestM%d'%vm, 'heartbeat')) - else: - heartbeat = 60 - f.write('heartbeat="%s"\n'% heartbeat) - elif re.match('(l2){0,1}swap.*\.cfg',config_file[-1]): - sutstatcores = cores[-1] - auto_start.append(True) - mach_type.append('sut') - elif re.match('secgw1.*\.cfg',config_file[-1]): - auto_start.append(True) + elif testconfig.has_option('TestM%d'%vm, 'dest_vm'): + # This must be a machine acting as a GW that we will not monitor mach_type.append('none') destVMindex = int(testconfig.get('TestM%d'%vm, 'dest_vm'))-1 f.write('dest_ip="%s"\n'% vmDPIP[machine_index[destVMindex]]) f.write('dest_hex_ip="%s"\n'% hexDPIP[machine_index[destVMindex]]) f.write('dest_hex_mac="%s"\n'% vmDPmac[machine_index[destVMindex]].replace(':',' ')) - elif re.match('secgw2.*\.cfg',config_file[-1]): + elif re.findall('mode\s*=\s*(swap|esp_dec)',filetext,re.MULTILINE): sutstatcores = cores[-1] - auto_start.append(True) mach_type.append('sut') else: - auto_start.append(True) mach_type.append('none') f.close tasks = tasks_for_this_cfg.union(tasks) @@ -1174,17 +1157,16 @@ for vm in range(0, int(required_number_of_test_machines)): clients[-1].scp_put('./{}_{}_parameters{}.lua'.format(env,test_file, vm+1), rundir + '/parameters.lua') if not configonly: if prox_launch_exit[vm]: - log.debug("Starting PROX on VM%d"%(vm+1)) - if auto_start[vm]: - cmd = 'sudo ' + rundir + '/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm] - else: + if mach_type[vm] == 'gen': cmd = 'sudo ' + rundir + '/prox -e -t -o cli -f ' + rundir + '/%s'%config_file[vm] + else: + cmd = 'sudo ' + rundir + '/prox -t -o cli -f ' + rundir + '/%s'%config_file[vm] + log.debug("Starting PROX on VM{}: {}".format((vm+1),cmd)) clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1)) socks_control.append(prox_launch_exit[vm]) socks.append(connect_socket(clients[-1])) sock_type.append(mach_type[vm]) sock_monitor.append(monitor[vm]) -atexit.register(exit_handler) monitor_gen = monitor_sut = False background_gen_socks =[] for index, sock in enumerate(socks, start=0): @@ -1205,9 +1187,6 @@ for index, sock in enumerate(socks, start=0): else: monitor_sut = True sutsock_index = index -if not(monitor_gen and monitor_sut): - log.exception("No generator and/or sut to monitor") - raise Exception("No generator and/or sut to monitor") if configonly: sys.exit() #################################################### -- cgit 1.2.3-korg