From f8ebc52a84ac662c6a2a5e8a4775d4a6217e7bb4 Mon Sep 17 00:00:00 2001 From: Christian Trautman Date: Tue, 12 Jul 2016 12:46:49 -0400 Subject: xena_deadlock: Fix xena2544.exe deadlock Fix Xena2544.exe execution with mono where sometimes the Xena process says complete, but the mono process doesn't exit. Changes call of Xena2544.exe from communicate thread lock to a wait try/catch block and checks the log periodically to see if a deadlock has occured and if so, the mono pid is terminated to prevent deadlock. * Fix one small issue with typecasting to int, changed to float JIRA: VSPERF-319 Change-Id: Ie7a86a2b9ff9cdfacba5a912ad17df6cc0581702 Signed-off-by: Christian Trautman --- tools/pkt_gen/xena/xena.py | 103 +++++++++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/tools/pkt_gen/xena/xena.py b/tools/pkt_gen/xena/xena.py index 7dd4b90b..67e9984f 100755 --- a/tools/pkt_gen/xena/xena.py +++ b/tools/pkt_gen/xena/xena.py @@ -25,6 +25,7 @@ Xena Traffic Generator Model # python imports import binascii import logging +import os import subprocess import sys from time import sleep @@ -50,6 +51,7 @@ from tools.pkt_gen.xena.XenaDriver import ( XenaManager, ) + class Xena(ITrafficGenerator): """ Xena Traffic generator wrapper class @@ -65,6 +67,19 @@ class Xena(ITrafficGenerator): self._duration = None self.tx_stats = None self.rx_stats = None + self._log_handle = None + + user_home = os.path.expanduser('~') + self._log_path = '{}/Xena/Xena2544-2G/Logs/xena2544.log'.format( + user_home) + + # make the folder and log file if they doesn't exist + if not os.path.exists(self._log_path): + os.makedirs(os.path.dirname(self._log_path)) + + # empty the file contents + open(self._log_path, 'w').close() + @property def traffic_defaults(self): @@ -99,7 +114,7 @@ class Xena(ITrafficGenerator): root[0][1][0][0].get('PortRxBpsL1')) + float( root[0][1][0][1].get('PortRxBpsL1')))/ 1000000 results[ResultsConstants.THROUGHPUT_RX_PERCENT] = ( - 100 - int(root[0][1][0].get('TotalLossRatioPcnt'))) * float( + 100 - float(root[0][1][0].get('TotalLossRatioPcnt'))) * float( root[0][1][0].get('TotalTxRatePcnt'))/100 results[ResultsConstants.TX_RATE_FPS] = root[0][1][0].get( 'TotalTxRateFps') @@ -451,6 +466,44 @@ class Xena(ITrafficGenerator): self.rx_stats = self.xmanager.ports[1].get_rx_stats() sleep(1) + def _start_xena_2544(self): + """ + Start the xena2544 exe. + :return: None + """ + args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c", + "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r", + "./tools/pkt_gen/xena", "-u", + settings.getValue('TRAFFICGEN_XENA_USER')] + # Sometimes Xena2544.exe completes, but mono holds the process without + # releasing it, this can cause a deadlock of the main thread. Use the + # xena log file as a way to detect this. + self._log_handle = open(self._log_path, 'r') + # read the contents of the log before we start so the next read in the + # wait method are only looking at the text from this test instance + self._log_handle.read() + self.mono_pipe = subprocess.Popen(args, stdout=sys.stdout) + + def _wait_xena_2544_complete(self): + """ + Wait for Xena2544.exe completion. + :return: None + """ + data = '' + while True: + try: + self.mono_pipe.wait(60) + self._log_handle.close() + break + except subprocess.TimeoutExpired: + # check the log to see if Xena2544 has completed and mono is + # deadlocked. + data += self._log_handle.read() + if 'TestCompletedSuccessfully' in data: + self._log_handle.close() + self.mono_pipe.terminate() + break + def _stop_api_traffic(self): """ Stop traffic through the socket API @@ -495,13 +548,11 @@ class Xena(ITrafficGenerator): See ITrafficGenerator for description """ self._duration = duration - self._params.clear() self._params['traffic'] = self.traffic_defaults.copy() if traffic: self._params['traffic'] = merge_spec(self._params['traffic'], traffic) - self._start_traffic_api(numpkts) return self._stop_api_traffic() @@ -517,7 +568,6 @@ class Xena(ITrafficGenerator): if traffic: self._params['traffic'] = merge_spec(self._params['traffic'], traffic) - self._start_traffic_api(-1) return self._stop_api_traffic() @@ -533,7 +583,6 @@ class Xena(ITrafficGenerator): if traffic: self._params['traffic'] = merge_spec(self._params['traffic'], traffic) - self._start_traffic_api(-1) def stop_cont_traffic(self): @@ -554,15 +603,10 @@ class Xena(ITrafficGenerator): if traffic: self._params['traffic'] = merge_spec(self._params['traffic'], traffic) - self._setup_json_config(trials, lossrate, '2544_throughput') + self._start_xena_2544() + self._wait_xena_2544_complete() - args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c", - "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r", - "./tools/pkt_gen/xena", "-u", - settings.getValue('TRAFFICGEN_XENA_USER')] - self.mono_pipe = subprocess.Popen(args, stdout=sys.stdout) - self.mono_pipe.communicate() root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot() return Xena._create_throughput_result(root) @@ -578,22 +622,15 @@ class Xena(ITrafficGenerator): if traffic: self._params['traffic'] = merge_spec(self._params['traffic'], traffic) - self._setup_json_config(trials, lossrate, '2544_throughput') - - args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c", - "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r", - "./tools/pkt_gen/xena", "-u", - settings.getValue('TRAFFICGEN_XENA_USER')] - self.mono_pipe = subprocess.Popen(args, stdout=sys.stdout) + self._start_xena_2544() def wait_rfc2544_throughput(self): """Wait for and return results of RFC2544 test. See ITrafficGenerator for description """ - self.mono_pipe.communicate() - sleep(2) + self._wait_xena_2544_complete() root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot() return Xena._create_throughput_result(root) @@ -610,16 +647,9 @@ class Xena(ITrafficGenerator): if traffic: self._params['traffic'] = merge_spec(self._params['traffic'], traffic) - self._setup_json_config(trials, lossrate, '2544_b2b') - - args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c", - "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r", - "./tools/pkt_gen/xena", "-u", - settings.getValue('TRAFFICGEN_XENA_USER')] - self.mono_pipe = subprocess.Popen( - args, stdout=sys.stdout) - self.mono_pipe.communicate() + self._start_xena_2544() + self._wait_xena_2544_complete() root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot() return Xena._create_throughput_result(root) @@ -630,27 +660,18 @@ class Xena(ITrafficGenerator): See ITrafficGenerator for description """ self._duration = duration - self._params.clear() self._params['traffic'] = self.traffic_defaults.copy() if traffic: self._params['traffic'] = merge_spec(self._params['traffic'], traffic) - self._setup_json_config(trials, lossrate, '2544_b2b') - - args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c", - "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r", - "./tools/pkt_gen/xena", "-u", - settings.getValue('TRAFFICGEN_XENA_USER')] - self.mono_pipe = subprocess.Popen( - args, stdout=sys.stdout) + self._start_xena_2544() def wait_rfc2544_back2back(self): """Wait and set results of RFC2544 test. """ - self.mono_pipe.communicate() - sleep(2) + self._wait_xena_2544_complete() root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot() return Xena._create_throughput_result(root) -- cgit 1.2.3-korg