run_traffic: capture and exit gracefully if crash in trex run_traffic

new non-blocking code of run_traffic has raise condition which causes
trex client to exit with STLError. This patches captures the exception
and exit gracefully

Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in
_bootstrap
self.run()
File "/usr/lib/python2.7/multiprocessing/process.py", line 114,
in run
self._target(*self._args, **self._kwargs)
File
"/opt/nsb_bin/yardstick_venv/lib/python2.7/site-packages/yardstick-0.1.dev0-py2.7.egg/yardstick/network_services/vnf_generic/vnf/sample_vnf.py",
line 945, in _traffic_runner
self.resource_helper.run_traffic(traffic_profile)
File
"/opt/nsb_bin/yardstick_venv/lib/python2.7/site-packages/yardstick-0.1.dev0-py2.7.egg/yardstick/network_services/vnf_generic/vnf/sample_vnf.py",
line 506, in run_traffic
self._run_traffic_once(traffic_profile)
File
"/opt/nsb_bin/yardstick_venv/lib/python2.7/site-packages/yardstick-0.1.dev0-py2.7.egg/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_trex.py",
line 63, in _run_traffic_once
self.client.stop(self.my_ports)
File
"/opt/nsb_bin/trex/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py",
line 1202, in wrap2
raise STLError("'{0}' - connection to the server had been lost: '{1}'".format(func_name, client.conn.get_disconnection_cause()))
STLError:

Change-Id: Ie5741339451e0a3f9c4bb48f64fd35d86d18e5d4
Signed-off-by: Deepak S <deepak.s@linux.intel.com>
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

3 files changed, 24 insertions, 15 deletions

diff --git a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
index af0d2ddde..07a862a8e 100644
--- a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
+++ b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
@@ -1142,7 +1142,7 @@ class TestClientResourceHelper(unittest.TestCase):
     }
 
     @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.LOG')
-    @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.STLStateError',
+    @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.STLError',
                 new_callable=lambda: MockError)
     def test_get_stats_not_connected(self, mock_state_error, mock_logger):
         vnfd_helper = VnfdHelper({})
diff --git a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
index 60979ebd2..659a7638c 100644
--- a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
+++ b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
@@ -39,7 +39,7 @@ from yardstick.network_services.utils import get_nsb_option
 
 from stl.trex_stl_lib.trex_stl_client import STLClient
 from stl.trex_stl_lib.trex_stl_client import LoggerApi
-from stl.trex_stl_lib.trex_stl_exceptions import STLError, STLStateError
+from stl.trex_stl_lib.trex_stl_exceptions import STLError
 
 from yardstick.ssh import AutoConnectSSH
 
@@ -457,7 +457,7 @@ class ClientResourceHelper(ResourceHelper):
     def get_stats(self, *args, **kwargs):
         try:
             return self.client.get_stats(*args, **kwargs)
-        except STLStateError:
+        except STLError:
             LOG.exception("TRex client not connected")
             return {}
 
@@ -497,18 +497,24 @@ class ClientResourceHelper(ResourceHelper):
     def run_traffic(self, traffic_profile):
         # fixme: fix passing correct trex config file,
         # instead of searching the default path
-        self._build_ports()
-        self.client = self._connect()
-        self.client.reset(ports=self.my_ports)
-        self.client.remove_all_streams(self.my_ports)  # remove all streams
-        traffic_profile.register_generator(self)
-
-        while self._terminated.value == 0:
-            self._run_traffic_once(traffic_profile)
-
-        self.client.stop(self.my_ports)
-        self.client.disconnect()
-        self._terminated.value = 0
+        try:
+            self._build_ports()
+            self.client = self._connect()
+            self.client.reset(ports=self.my_ports)
+            self.client.remove_all_streams(self.my_ports)  # remove all streams
+            traffic_profile.register_generator(self)
+
+            while self._terminated.value == 0:
+                self._run_traffic_once(traffic_profile)
+
+            self.client.stop(self.my_ports)
+            self.client.disconnect()
+            self._terminated.value = 0
+        except STLError:
+            if self._terminated.value:
+                LOG.debug("traffic generator is stopped")
+                return  # return if trex/tg server is stopped.
+            raise
 
     def terminate(self):
         self._terminated.value = 1  # stop client
diff --git a/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_trex.py b/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_trex.py
index 79e42e0a8..548060849 100644
--- a/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_trex.py
+++ b/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_trex.py
@@ -57,6 +57,9 @@ class TrexRfcResourceHelper(TrexResourceHelper):
         self.my_ports = list(set(chain(self.priv_ports, self.pub_ports)))
 
     def _run_traffic_once(self, traffic_profile):
+        if self._terminated.value:
+            return
+
         traffic_profile.execute(self)
         self.client_started.value = 1
         time.sleep(self.RUN_DURATION)