diff options
| author |   Luc Provoost <luc.provoost@intel.com> | 2020-12-16 15:47:49 +0100 |
|---|---|---|
| committer | Luc Provoost <luc.provoost@intel.com> | 2020-12-28 12:11:29 +0100 |
| commit | 9822834527c84e6e2d36b8b6d9aa81d0facd2a8a (patch) | |
| tree | 2562c9be8fd2d78d7f8eb3530e4df5802478479e /VNFs/DPPD-PROX | |
| parent | 63eeeaf4c2b27db28a0c4fb0c5806978b0a5796d (diff) | |
Improved PROX cleanup when exiting
Sending now a 'quit' command to PROX, so that the thread running PROX in
an ssh session is now cleaned up correctly.
Also fixed a problem with the logging when running the code more than
once. We only create handlers now, when they do not yet exist.
Updated the rapid version.
Also added some more tests in the testcases.yaml for xtesting.
At the end of a run, the PROX.log files are copied in the results
directory.
The success criterium for a test 'pass_threshold' has been removed from
the test files and is to be controlled by the xtesting testcases.yaml
file.
Change-Id: Ifbbb1c91f32c9176f52025d9ae4c495b432a94c9
Signed-off-by: Luc Provoost <luc.provoost@intel.com>
Diffstat (limited to 'VNFs/DPPD-PROX')
23 files changed, 232 insertions, 167 deletions
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput.test b/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput.test index ce9ba72b..6b1e6772 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput.test +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput.test @@ -42,11 +42,6 @@ warmuptime=2 [test2] test=TST009test -# Following parameter defines the success criterium for the test. -# When this test uses multiple combinations of packet size and flows, -# all combinations must be meeting the same threshold -# The threshold is expressed in Mpps -pass_threshold=0.1 imixs=[[64],[128]] # the number of flows in the list need to be powers of 2, max 2^20 # Select from following numbers: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576 diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput_64B_64F.test b/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput_64B_64F.test index 866db5c3..c10c50e7 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput_64B_64F.test +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput_64B_64F.test @@ -35,11 +35,6 @@ cores = [1] [test1] test=TST009test -# Following parameter defines the success criterium for the test. -# When this test uses multiple combinations of packet size and flows, -# all combinations must be meeting the same threshold -# The threshold is expressed in Mpps -pass_threshold=0.001 imixs=[[64]] # the number of flows in the list need to be powers of 2, max 2^20 # Select from following numbers: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576 diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput_acaeab_16384F.test b/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput_acaeab_16384F.test new file mode 100644 index 00000000..9b6585a6 --- /dev/null +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/TST009_Throughput_acaeab_16384F.test @@ -0,0 +1,54 @@ +## +## Copyright (c) 2010-2020 Intel Corporation +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +[TestParameters] +name = Rapid_ETSINFV_TST009 +number_of_tests = 1 +total_number_of_test_machines = 2 +lat_percentile = 99 + +[TestM1] +name = Generator +config_file = gen.cfg +dest_vm = 2 +gencores = [1] +latcores = [3] +#bucket_size_exp = 12 + +[TestM2] +name = Swap +config_file = swap.cfg +cores = [1] + +[test1] +test=TST009test +# Following parameter defines the success criterium for the test. +# When this test uses multiple combinations of packet size and flows, +# all combinations must be meeting the same threshold +# The threshold is expressed in Mpps +pass_threshold=0.001 +imixs=[[64,256,64,1024,64,128]] +# the number of flows in the list need to be powers of 2, max 2^20 +# Select from following numbers: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576 +flows=[16384] +drop_rate_threshold = 0 +lat_avg_threshold = 120 +lat_perc_threshold = 220 +lat_max_threshold = inf +MAXr = 3 +MAXz = 5000 +MAXFramesPerSecondAllIngress = 12000000 +StepSize = 10000 diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/basicrapid.test b/VNFs/DPPD-PROX/helper-scripts/rapid/basicrapid.test index fbf75a02..f27e4987 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/basicrapid.test +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/basicrapid.test @@ -44,11 +44,6 @@ warmuptime=2 [test2] test=flowsizetest -# Following parameter defines the success criterium for the test. -# When this test uses multiple combinations of packet size and flows, -# all combinations must be meeting the same threshold -# The threshold is expressed in Mpps -pass_threshold=0.1 # Each element in the imix list will result in a separate test. Each element # is on its turn a list of packet sizes which will be used during one test # execution. If you only want to test 1 size, define a list with only one diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/format.yaml b/VNFs/DPPD-PROX/helper-scripts/rapid/format.yaml index 6b1eb456..d9c55405 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/format.yaml +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/format.yaml @@ -22,6 +22,8 @@ PacketsReceived: PacketsReceived PacketsLost: PacketsLost rapid_flowsizetest: + Environment: environment_file + Test: test Flows: Flows Size: Size Speed (Mpps): @@ -42,17 +44,7 @@ rapid_flowsizetest: PacketsReceived: PacketsReceived PacketsLost: PacketsLost rapid_irqtest: - Core: Core - LessThan1us : B1 - LessThan5us : B5 - LessThan10us : B10 - LessThan50us : B50 - LessThan100us : B100 - LessThan500us : B500 - LessThan1ms : B1000 - LessThan5ms : B5000 - LessThan10ms : B10000 - LessThan50ms : B50000 - LessThan100ms : B100000 - LessThan500ms : B500000 - MoreThan500ms : BM500000 + Environment: environment_file + Test: test + Buckets: buckets + Machine_data: machine_data diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/increment_till_fail.test b/VNFs/DPPD-PROX/helper-scripts/rapid/increment_till_fail.test index 29e36bfd..d07876be 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/increment_till_fail.test +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/increment_till_fail.test @@ -44,11 +44,6 @@ warmuptime=2 [test2] test=increment_till_fail -# Following parameter defines the success criterium for the test. -# When this test uses multiple combinations of packet size and flows, -# all combinations must be meeting the same threshold -# The threshold is expressed in Mpps -pass_threshold=0.1 # Each element in the imix list will result in a separate test. Each element # is on its turn a list of packet sizes which will be used during one test # execution. If you only want to test 1 size, define a list with only one diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/ipv6.test b/VNFs/DPPD-PROX/helper-scripts/rapid/ipv6.test index b9f00f71..7dd586af 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/ipv6.test +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/ipv6.test @@ -45,12 +45,8 @@ warmuptime=2 [test2] test=flowsizetest -# Following parameter defines the success criterium for the test. -# When this test uses multiple combinations of packet size and flows, -# all combinations must be meeting the same threshold -# The threshold is expressed in Mpps -pass_threshold=0.1 -# DO NOT USE IMIX FOR IPV6 TESTING +# DO NOT USE IMIX FOR IPV6 TESTING. THE LIST OF IMIXS CAN ONLY CONTAIN LISTS +# WITH ONE ELEMENT!!! # PACKET SIZE NEEDS TO BE AT LEAST 84 (66 + 18) FOR IPV6 # 18 bytes needed for UDP LATENCY AND COUNTER CONTENT imixs=[[84],[250]] diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/l2framerate.test b/VNFs/DPPD-PROX/helper-scripts/rapid/l2framerate.test index e09b80f3..0c868006 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/l2framerate.test +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/l2framerate.test @@ -33,11 +33,6 @@ cores = [1] [test1] test=fixed_rate -# Following parameter defines the success criterium for the test. -# When this test uses multiple combinations of packet size and flows, -# all combinations must be meeting the same threshold -# The threshold is expressed in Mpps -pass_threshold=0.1 startspeed = 10 imixs=[[256]] flows=[64] diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/l2zeroloss.test b/VNFs/DPPD-PROX/helper-scripts/rapid/l2zeroloss.test index 2f61df56..bc1a1bca 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/l2zeroloss.test +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/l2zeroloss.test @@ -40,11 +40,6 @@ warmuptime=2 [test2] test=flowsizetest -# Following parameter defines the success criterium for the test. -# When this test uses multiple combinations of packet size and flows, -# all combinations must be meeting the same threshold -# The threshold is expressed in Mpps -pass_threshold=0.1 # Each element in the imix list will result in a separate test. Each element # is on its turn a list of packet sizes which will be used during one test # execution. If you only want to test 1 size, define a list with only one diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/l3framerate.test b/VNFs/DPPD-PROX/helper-scripts/rapid/l3framerate.test index 1d890d13..67a57ce7 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/l3framerate.test +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/l3framerate.test @@ -40,11 +40,6 @@ warmuptime=2 [test2] test=fixed_rate -# Following parameter defines the success criterium for the test. -# When this test uses multiple combinations of packet size and flows, -# all combinations must be meeting the same threshold -# The threshold is expressed in Mpps -pass_threshold=0.1 imixs=[[64],[128]] # the number of flows in the list need to be powers of 2, max 2^20 # If not a power of 2, we will use the lowest power of 2 that is larger than diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py index 586731eb..5fc98db3 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/prox_ctrl.py @@ -183,9 +183,8 @@ class prox_sock(object): self._sock = sock self._rcvd = b'' - def quit(self): + def __del__(self): if self._sock is not None: - self._send('quit') self._sock.close() self._sock = None @@ -321,6 +320,9 @@ class prox_sock(object): self._send('set value %s %s %s %s %s' % (','.join(map(str, cores)), task, offset, value, length)) + def quit_prox(self): + self._send('quit') + def _send(self, cmd): """Append LF and send command to the PROX instance.""" if self._sock is None: diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_defaults.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_defaults.py index 404e6e32..e648c016 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_defaults.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_defaults.py @@ -21,7 +21,7 @@ class RapidDefaults(object): Class to define the test defaults """ test_params = { - 'version' : '2020.09.23', # Please do NOT change, used for debugging + 'version' : '2020.12.21', # Please do NOT change, used for debugging 'environment_file' : 'rapid.env', #Default string for environment 'test_file' : 'basicrapid.test', #Default string for test 'machine_map_file' : 'machine.map', #Default string for machine map file @@ -30,5 +30,6 @@ class RapidDefaults(object): 'runtime' : 10, # time in seconds for 1 test run 'configonly' : False, # If True, the system will upload all the necessary config fiels to the VMs, but not start PROX and the actual testing 'rundir' : '/opt/rapid', # Directory where to find the tools in the machines running PROX + 'resultsdir' : '.', # Directory where to store log files 'lat_percentile' : 0.99 } diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py index c4308b1f..566e7cf7 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_flowsizetest.py @@ -99,7 +99,7 @@ class FlowSizeTest(RapidTest): def run(self): result_details = {'Details': 'Nothing'} self.gen_machine.start_latency_cores() - TestPassed = True + TestResult = 0 for imix in self.test['imixs']: size = mean(imix) self.gen_machine.set_udp_packet_size(imix) @@ -122,7 +122,6 @@ class FlowSizeTest(RapidTest): self.set_background_flows(self.background_machines, flow_number) endspeed = None speed = self.get_start_speed_and_init(size) - self.record_start_time() while True: attempts += 1 endwarning = False @@ -164,7 +163,7 @@ class FlowSizeTest(RapidTest): if lat_warning or retry_warning: endwarning = '| | {:177.177} |'.format(retry_warning + lat_warning) success = True - TestPassed = False # fixed rate testing cannot be True, it is just reporting numbers every second + TestResult = TestResult + pps_rx # fixed rate testing result is strange: we just report the pps received speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC # The following if statement is testing if we pass the success criteria of a certain drop rate, average latency and maximum latency below the threshold # The drop rate success can be achieved in 2 ways: either the drop rate is below a treshold, either we want that no packet has been lost during the test @@ -234,10 +233,7 @@ class FlowSizeTest(RapidTest): break elif self.resolution_achieved(): break - self.record_stop_time() if endspeed is not None: - if TestPassed and (endpps_rx < self.test['pass_threshold']): - TestPassed = False speed_prefix = lat_avg_prefix = lat_perc_prefix = lat_max_prefix = abs_drop_rate_prefix = drop_rate_prefix = bcolors.ENDC RapidLog.info(self.report_result(flow_number,size,endspeed,endpps_req_tx,endpps_tx,endpps_sut_tx,endpps_rx,endlat_avg,endlat_perc,endlat_perc_max,endlat_max,endabs_tx,endabs_rx,endabs_dropped,actual_duration,speed_prefix,lat_avg_prefix,lat_perc_prefix,lat_max_prefix,abs_drop_rate_prefix,drop_rate_prefix)) if endavg_bg_rate: @@ -248,10 +244,9 @@ class FlowSizeTest(RapidTest): RapidLog.info (endwarning) RapidLog.info("+--------+------------------+-------------+-------------+-------------+------------------------+----------+----------+----------+-----------+-----------+-----------+-----------+-------+----+") if self.test['test'] != 'fixed_rate': + TestResult = TestResult + endpps_rx result_details = {'test': self.test['testname'], 'environment_file': self.test['environment_file'], - 'start_date': self.start, - 'stop_date': self.stop, 'Flows': flow_number, 'Size': size, 'RequestedSpeed': RapidTest.get_pps(endspeed,size), @@ -271,4 +266,4 @@ class FlowSizeTest(RapidTest): else: RapidLog.info('|{:>7}'.format(str(flow_number))+" | Speed 0 or close to 0") self.gen_machine.stop_latency_cores() - return (TestPassed, result_details) + return (TestResult, result_details) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py index 293720de..99cb361e 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py @@ -49,7 +49,8 @@ class RapidGeneratorMachine(RapidMachine): """ Class to deal with a generator PROX instance (VM, bare metal, container) """ - def __init__(self, key, user, vim, rundir, machine_params, configonly, ipv6): + def __init__(self, key, user, vim, rundir, resultsdir, machine_params, + configonly, ipv6): mac_address_size = 6 ethertype_size = 2 FCS_size = 4 @@ -73,7 +74,8 @@ class RapidGeneratorMachine(RapidMachine): self.udp_dest_port_offset = udp_header_start_offset + 2 self.udp_length_offset = udp_header_start_offset + 4 self.ipv6 = ipv6 - super().__init__(key, user, vim, rundir, machine_params, configonly) + super().__init__(key, user, vim, rundir, resultsdir, machine_params, + configonly) def get_cores(self): return (self.machine_params['gencores'] + diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_irqtest.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_irqtest.py index 1afa0f19..f990a230 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_irqtest.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_irqtest.py @@ -34,16 +34,24 @@ class IrqTest(RapidTest): self.machines = machines def run(self): - RapidLog.info("+----------------------------------------------------------------------------------------------------------------------------") - RapidLog.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic ") - RapidLog.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and ") - RapidLog.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was ") - RapidLog.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout ") - RapidLog.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 ") - RapidLog.info("+----------------------------------------------------------------------------------------------------------------------------") + RapidLog.info("+----------------------------------------------------------------------------------------------------------------------------+") + RapidLog.info("| Measuring time probably spent dealing with an interrupt. Interrupting DPDK cores for more than 50us might be problematic |") + RapidLog.info("| and result in packet loss. The first row shows the interrupted time buckets: first number is the bucket between 0us and |") + RapidLog.info("| that number expressed in us and so on. The numbers in the other rows show how many times per second, the program was |") + RapidLog.info("| interrupted for a time as specified by its bucket. '0' is printed when there are no interrupts in this bucket throughout |") + RapidLog.info("| the duration of the test. 0.00 means there were interrupts in this bucket but very few. Due to rounding this shows as 0.00 |") + RapidLog.info("+----------------------------------------------------------------------------------------------------------------------------+") sys.stdout.flush() + max_loop_duration = 0 + machine_details = {} for machine in self.machines: buckets=machine.socket.show_irq_buckets(1) + if max_loop_duration == 0: + # First time we go through the loop, we need to initialize + # result_details + result_details = {'test': self.test['testname'], + 'environment_file': self.test['environment_file'], + 'buckets': buckets} print('Measurement ongoing ... ',end='\r') machine.start() # PROX cores will be started within 0 to 1 seconds # That is why we sleep a bit over 1 second to make sure all cores @@ -60,7 +68,7 @@ class IrqTest(RapidTest): old_irq[i][j] = machine.socket.irq_stats(irqcore,j) # Measurements in the loop above, are updated by PROX every second # This means that taking the same measurement 0.5 second later - # might results in the same data or data from the next 1s window + # might result in the same data or data from the next 1s window time.sleep(float(self.test['runtime'])) row_names = [] for i,irqcore in enumerate(machine.get_cores()): @@ -70,10 +78,13 @@ class IrqTest(RapidTest): if diff == 0: irq[i][j] = '0' else: - irq[i][j] = str(round(old_div(diff,float(self.test['runtime'])), 2)) + irq[i][j] = str(round(old_div(diff, + float(self.test['runtime'])), 2)) + if max_loop_duration < int(bucket): + max_loop_duration = int(bucket) # Measurements in the loop above, are updated by PROX every second # This means that taking the same measurement 0.5 second later - # might results in the same data or data from the next 1s window + # might result in the same data or data from the next 1s window # Conclusion: we don't know the exact window size. # Real measurement windows might be wrong by 1 second # This could be fixed in this script by checking this data every @@ -81,17 +92,15 @@ class IrqTest(RapidTest): # a longer time and decrease the error. The absolute number of # interrupts is not so important. machine.stop() + core_details = {} RapidLog.info('Results for PROX instance %s'%machine.name) - RapidLog.info('{:>12}'.format('bucket us') + ''.join(['{:>12}'.format(item) for item in column_names])) + RapidLog.info('{:>12}'.format('bucket us') + + ''.join(['{:>12}'.format(item) for item in column_names])) for j, row in enumerate(irq): - RapidLog.info('Core {:>7}'.format(row_names[j]) + ''.join(['{:>12}'.format(item) for item in row])) - variables = {} - variables['test'] = self.test['test'] - variables['environment_file'] = self.test['environment_file'] - variables['Machine'] = machine.name - for i,irqcore in enumerate(machine.get_cores()): - variables['Core'] = '{}'.format(row_names[i]) - for j,bucket in enumerate(buckets): - variables['B{}'.format(column_names[j].replace(">","M").replace("<","").replace(" ",""))] = irq[i][j] - self.post_data('rapid_irqtest', variables) - return (True, None) + RapidLog.info('Core {:>7}'.format(row_names[j]) + + ''.join(['{:>12}'.format(item) for item in row])) + core_details['Core {}'.format(row_names[j])] = row + machine_details[machine.name] = core_details + result_details['machine_data'] = machine_details + result_details = self.post_data('rapid_irqtest', result_details) + return (500000 - max_loop_duration, result_details) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_log.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_log.py index d1460f55..f453c574 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_log.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_log.py @@ -42,56 +42,67 @@ class RapidLog(object): @staticmethod def log_init(log_file, loglevel, screenloglevel, version): - # create formatters - screen_formatter = logging.Formatter("%(message)s") - file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") - - # get a top-level logger, - # set its log level, - # BUT PREVENT IT from propagating messages to the root logger - # - log = logging.getLogger() - numeric_level = getattr(logging, loglevel.upper(), None) - if not isinstance(numeric_level, int): - raise ValueError('Invalid log level: %s' % loglevel) - log.setLevel(numeric_level) - log.propagate = 0 - - # create a console handler - # and set its log level to the command-line option - # - console_handler = logging.StreamHandler(sys.stdout) - #console_handler.setLevel(logging.INFO) - numeric_screenlevel = getattr(logging, screenloglevel.upper(), None) - if not isinstance(numeric_screenlevel, int): - raise ValueError('Invalid screenlog level: %s' % screenloglevel) - console_handler.setLevel(numeric_screenlevel) - console_handler.setFormatter(screen_formatter) - - # create a file handler - # and set its log level - # - file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10) - #file_handler = log.handlers.TimedRotatingFileHandler(log_file, 'D', 1, 5) - file_handler.setLevel(numeric_level) - file_handler.setFormatter(file_formatter) - - # add handlers to the logger - # - log.addHandler(file_handler) - log.addHandler(console_handler) - - # Check if log exists and should therefore be rolled - needRoll = os.path.isfile(log_file) - - - # This is a stale log, so roll it - if needRoll: - # Add timestamp - log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime()) - - # Roll over on application start - file_handler.doRollover() + log = logging.getLogger(__name__) + makeFileHandler = True + makeStreamHandler = True + if len(log.handlers) > 0: + for handler in log.handlers: + if isinstance(handler, logging.FileHandler): + makeFileHandler = False + elif isinstance(handler, logging.StreamHandler): + makeStreamHandler = False + if makeStreamHandler: + # create formatters + screen_formatter = logging.Formatter("%(message)s") + # create a console handler + # and set its log level to the command-line option + # + console_handler = logging.StreamHandler(sys.stdout) + #console_handler.setLevel(logging.INFO) + numeric_screenlevel = getattr(logging, screenloglevel.upper(), None) + if not isinstance(numeric_screenlevel, int): + raise ValueError('Invalid screenlog level: %s' % screenloglevel) + console_handler.setLevel(numeric_screenlevel) + console_handler.setFormatter(screen_formatter) + # add handler to the logger + # + log.addHandler(console_handler) + if makeFileHandler: + # create formatters + file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") + # get a top-level logger, + # set its log level, + # BUT PREVENT IT from propagating messages to the root logger + # + numeric_level = getattr(logging, loglevel.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: %s' % loglevel) + log.setLevel(numeric_level) + log.propagate = 0 + + + # create a file handler + # and set its log level + # + file_handler = logging.handlers.RotatingFileHandler(log_file, backupCount=10) + file_handler.setLevel(numeric_level) + file_handler.setFormatter(file_formatter) + + # add handler to the logger + # + log.addHandler(file_handler) + + # Check if log exists and should therefore be rolled + needRoll = os.path.isfile(log_file) + + + # This is a stale log, so roll it + if needRoll: + # Add timestamp + log.debug('\n---------\nLog closed on %s.\n---------\n' % time.asctime()) + + # Roll over on application start + file_handler.doRollover() # Add timestamp log.debug('\n---------\nLog started on %s.\n---------\n' % time.asctime()) @@ -100,6 +111,13 @@ class RapidLog(object): RapidLog.log = log @staticmethod + def log_close(): + for handler in RapidLog.log.handlers: + if isinstance(handler, logging.FileHandler): + handler.close() + RapidLog.log.removeHandler(handler) + + @staticmethod def exception(exception_info): RapidLog.log.exception(exception_info) raise Exception(exception_info) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py index 8466c856..e47c1799 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py @@ -24,12 +24,14 @@ class RapidMachine(object): """ Class to deal with a PROX instance (VM, bare metal, container) """ - def __init__(self, key, user, vim, rundir, machine_params, configonly): + def __init__(self, key, user, vim, rundir, resultsdir, machine_params, + configonly): self.name = machine_params['name'] self.ip = machine_params['admin_ip'] self.key = key self.user = user self.rundir = rundir + self.resultsdir = resultsdir self.dp_ports = [] self.dpdk_port_index = [] self.configonly = configonly @@ -44,13 +46,13 @@ class RapidMachine(object): index += 1 else: break - self.rundir = rundir self.machine_params = machine_params self.vim = vim def __del__(self): if ((not self.configonly) and self.machine_params['prox_socket']): - self._client.scp_get('/prox.log', './{}.prox.log'.format(self.name)) + self._client.scp_get('/prox.log', '{}/{}.prox.log'.format( + self.resultsdir, self.name)) def get_cores(self): return (self.machine_params['cores']) @@ -113,7 +115,7 @@ class RapidMachine(object): def close_prox(self): if (not self.configonly) and self.machine_params['prox_socket'] and self.machine_params['prox_launch_exit']: - self.socket.quit() + self.socket.quit_prox() def connect_prox(self): if self.machine_params['prox_socket']: diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py index 29d87556..e5ed10d0 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py @@ -73,7 +73,7 @@ class RapidConfigParser(object): elif option in ['startspeed', 'step', 'drop_rate_threshold', 'lat_avg_threshold','lat_perc_threshold', 'lat_max_threshold','accuracy','maxr','maxz', - 'pass_threshold','ramp_step']: + 'ramp_step']: test[option] = float(testconfig.get(section, option)) else: test[option] = testconfig.get(section, option) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py index b89eb7bc..a54caba5 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_test.py @@ -105,12 +105,6 @@ class RapidTest(object): if v in variables.keys(): data_format[k] = variables[v] - def record_start_time(self): - self.start = dt.now().strftime('%Y-%m-%d %H:%M:%S') - - def record_stop_time(self): - self.stop = dt.now().strftime('%Y-%m-%d %H:%M:%S') - def post_data(self, test, variables): var = copy.deepcopy(self.data_format) self.parse_data_format_dict(var, variables) diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapidxt.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapidxt.py index b472f367..2a82df5c 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/rapidxt.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/rapidxt.py @@ -36,6 +36,7 @@ class RapidXt(testcase.TestCase): for key in kwargs: test_params[key] = kwargs[key] os.makedirs(self.res_dir, exist_ok=True) + test_params['resultsdir'] = self.res_dir log_file = '{}/RUN{}.{}.log'.format(self.res_dir, test_params['environment_file'], test_params['test_file']) RapidLog.log_init(log_file, test_params['loglevel'], @@ -43,9 +44,9 @@ class RapidXt(testcase.TestCase): test_manager = RapidTestManager() self.start_time = time.time() self.result, self.details = test_manager.run_tests(test_params) - self.result = 100 * self.result - RapidLog.info('Test result is : {}'.format(self.result)) self.stop_time = time.time() + RapidLog.log_close() + except Exception: # pylint: disable=broad-except print("Unexpected error:", sys.exc_info()[0]) self.result = 0 diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py index 44f33c06..3bdb91d1 100755 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/runrapid.py @@ -62,8 +62,8 @@ class RapidTestManager(object): if 'gencores' in machine_params.keys(): machine = RapidGeneratorMachine(test_params['key'], test_params['user'], test_params['vim_type'], - test_params['rundir'], machine_params, - configonly, test_params['ipv6']) + test_params['rundir'], test_params['resultsdir'], + machine_params, configonly, test_params['ipv6']) if machine_params['monitor']: if monitor_gen: RapidLog.exception("Can only monitor 1 generator") @@ -76,7 +76,7 @@ class RapidTestManager(object): else: machine = RapidMachine(test_params['key'], test_params['user'], test_params['vim_type'], test_params['rundir'], - machine_params, configonly) + test_params['resultsdir'], machine_params, configonly) if machine_params['monitor']: if monitor_sut: RapidLog.exception("Can only monitor 1 sut") @@ -94,7 +94,7 @@ class RapidTestManager(object): with concurrent.futures.ThreadPoolExecutor(max_workers=len(self.machines)) as executor: future_to_connect_prox = {executor.submit(machine.connect_prox): machine for machine in self.machines} concurrent.futures.wait(future_to_connect_prox,return_when=ALL_COMPLETED) - result = True + result = 0 for test_param in test_params['tests']: RapidLog.info(test_param['test']) if test_param['test'] in ['flowsizetest', 'TST009test', @@ -128,8 +128,10 @@ class RapidTestManager(object): RapidLog.debug('Test name ({}) is not valid:'.format( test_param['test'])) single_test_result, result_details = test.run() - if not single_test_result: - result = False + result = result + single_test_result + for machine in self.machines: + machine.close_prox() + concurrent.futures.wait(self.future_to_prox,return_when=ALL_COMPLETED) return (result, result_details) def main(): @@ -145,7 +147,7 @@ def main(): test_params['screenloglevel'] , test_params['version'] ) test_manager = RapidTestManager() test_result, _ = test_manager.run_tests(test_params) - RapidLog.info('Test result is : {}'.format(test_result)) + RapidLog.log_close() if __name__ == "__main__": main() diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/xtesting/Dockerfile b/VNFs/DPPD-PROX/helper-scripts/rapid/xtesting/Dockerfile index b7bced8d..299f8ef3 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/xtesting/Dockerfile +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/xtesting/Dockerfile @@ -24,7 +24,6 @@ RUN git clone https://git.opnfv.org/samplevnf /samplevnf WORKDIR /samplevnf/VNFs/DPPD-PROX/helper-scripts/rapid COPY rapid.env /samplevnf/VNFs/DPPD-PROX/helper-scripts/rapid/. COPY rapid_key.pem /samplevnf/VNFs/DPPD-PROX/helper-scripts/rapid/. -COPY TST009_Throughput_64B_64F.test /samplevnf/VNFs/DPPD-PROX/helper-scripts/rapid/. COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml RUN apk add python3-dev openssh-client && cd /samplevnf/VNFs/DPPD-PROX/helper-scripts/rapid/ && git init && pip3 install . CMD ["run_tests", "-t", "all"] diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/xtesting/testcases.yaml b/VNFs/DPPD-PROX/helper-scripts/rapid/xtesting/testcases.yaml index 2db064f2..38ba1314 100644 --- a/VNFs/DPPD-PROX/helper-scripts/rapid/xtesting/testcases.yaml +++ b/VNFs/DPPD-PROX/helper-scripts/rapid/xtesting/testcases.yaml @@ -1,18 +1,51 @@ --- tiers: - - name: rapid + name: IRQ_rapid_benchmarking order: 1 - description: 'Rapid Testing' + description: 'IRQ Rapid Testing' testcases: - - case_name: rapid_tst009 + case_name: rapid_irq project_name: rapidxt - criteria: 100 + criteria: 499500 + # Criterium for irq is defined as 500000 - the maximal allowed interrupt time per PMD loop (in us) + blocking: true + clean_flag: false + description: 'IRQ test' + run: + name: rapidxt + args: + test_file: irq.test + runtime: 5 + - + name: TST009_rapid_benchmarking + order: 2 + description: 'TST009 Rapid Testing' + testcases: + - + case_name: rapid_tst009_64b_64f + project_name: rapidxt + criteria: 0.5 + # Criterium for TST009 testing is defined as the minimum packets per second received in the generator, expressed in Mpps blocking: true clean_flag: false description: 'TST009 test, 64 byte packets, 64 flows' run: name: rapidxt args: - test_file: TST009_Throughput_64B_64F.test + test_file: TST009_Throughput_64B_64F.test + runtime: 5 + - + case_name: rapid_tst009_acaeab_16384f + project_name: rapidxt + criteria: 0.2 + # Criterium for TST009 testing is defined as the minimum packets per second received in the generator, expressed in Mpps + blocking: true + clean_flag: false + description: 'TST009 test, imix acaeab, 16384 flows' + run: + name: rapidxt + args: + test_file: TST009_Throughput_acaeab_16384F.test + runtime: 5 |