Add possibility to restart TRex in case of config change or forced it with config flag

Change-Id: I40473eac355b76655220d48062eff851cc4eebc3
Signed-off-by: fmenguy <francoisregis.menguy@orange.com>

6 files changed, 122 insertions, 31 deletions

diff --git a/docs/testing/user/userguide/advanced.rst b/docs/testing/user/userguide/advanced.rst
index 1d2ac36..1a6e999 100644
--- a/docs/testing/user/userguide/advanced.rst
+++ b/docs/testing/user/userguide/advanced.rst
@@ -78,6 +78,22 @@ Used parameters:
 * ``--no-traffic`` or ``-0`` : sending traffic from traffic generator is skipped
 
 
+TRex force restart
+------------------------------------
+
+NFVbench allows to restart TRex traffic generator between runs.
+It runs the whole test, but restart TRex instance before generating new traffic.
+
+To force restart, use the --restart option:
+
+.. code-block:: bash
+
+    nfvbench --restart
+
+Used parameters:
+
+* ``--restart`` : restart traffic generator (TRex)
+
 
 Fixed Rate Run
 --------------
diff --git a/docs/testing/user/userguide/server.rst b/docs/testing/user/userguide/server.rst
index 806927b..fc56dfc 100644
--- a/docs/testing/user/userguide/server.rst
+++ b/docs/testing/user/userguide/server.rst
@@ -311,6 +311,15 @@ A short run of 5 seconds at a fixed rate of 1Mpps (and everything else same as t
        "rate": "1Mpps"
     }
 
+Use the default configuration but force TRex restart:
+
+.. code-block:: bash
+
+    {
+       "restart": true
+    }
+
+
 Example of interaction with the NFVbench server using HTTP and curl
 -------------------------------------------------------------------
 HTTP requests can be sent directly to the NFVbench server from CLI using curl from any host that can connect to the server (here we run it from the local host).
diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml
index 5d12e39..9fc7ae4 100755
--- a/nfvbench/cfg.default.yaml
+++ b/nfvbench/cfg.default.yaml
@@ -276,6 +276,10 @@ traffic_generator:
               - socket:
                 threads:
 
+# Use 'true' to force restart of local TRex server before next run
+# TRex local server will be restarted even if restart property is false in case of generator config changes between runs
+restart: false
+
 # Simpler override for trex core count and mbuf multilier factor
 # if empty defaults to the one specified in generator_profile.cores
 cores:
diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py
index bb73d68..e585154 100644
--- a/nfvbench/nfvbench.py
+++ b/nfvbench/nfvbench.py
@@ -354,6 +354,11 @@ def _parse_opts_from_cli():
                         action='store_true',
                         help='Cleanup NFVbench resources (do not prompt)')
 
+    parser.add_argument('--restart', dest='restart',
+                        default=None,
+                        action='store_true',
+                        help='Restart TRex server')
+
     parser.add_argument('--json', dest='json',
                         action='store',
                         help='store results in json format file',
@@ -552,7 +557,8 @@ def main():
             config.compute_nodes = opts.hypervisor
         if opts.vxlan:
             config.vxlan = True
-
+        if opts.restart:
+            config.restart = True
         # port to port loopback (direct or through switch)
         if opts.l2_loopback:
             config.l2_loopback = True
diff --git a/nfvbench/traffic_gen/trex_gen.py b/nfvbench/traffic_gen/trex_gen.py
index bbb67c1..daf5fb1 100644
--- a/nfvbench/traffic_gen/trex_gen.py
+++ b/nfvbench/traffic_gen/trex_gen.py
@@ -440,30 +440,13 @@ class TRex(AbstractTrafficGenerator):
                                 async_port=self.generator_config.zmq_pub_port)
         try:
             self.__connect(self.client)
+            if server_ip == '127.0.0.1':
+                config_updated = self.__check_config()
+                if config_updated or self.config.restart:
+                    self.__restart()
         except (TimeoutError, STLError) as e:
             if server_ip == '127.0.0.1':
-                try:
-                    self.__start_server()
-                    self.__connect_after_start()
-                except (TimeoutError, STLError) as e:
-                    LOG.error('Cannot connect to TRex')
-                    LOG.error(traceback.format_exc())
-                    logpath = '/tmp/trex.log'
-                    if os.path.isfile(logpath):
-                        # Wait for TRex to finish writing error message
-                        last_size = 0
-                        for _ in xrange(self.config.generic_retry_count):
-                            size = os.path.getsize(logpath)
-                            if size == last_size:
-                                # probably not writing anymore
-                                break
-                            last_size = size
-                            time.sleep(1)
-                        with open(logpath, 'r') as f:
-                            message = f.read()
-                    else:
-                        message = e.message
-                    raise TrafficGeneratorException(message)
+                self.__start_local_server()
             else:
                 raise TrafficGeneratorException(e.message)
 
@@ -498,10 +481,63 @@ class TRex(AbstractTrafficGenerator):
                                             (self.port_info[0]['speed'],
                                              self.port_info[1]['speed']))
 
+    def __start_local_server(self):
+        try:
+            LOG.info("Starting TRex ...")
+            self.__start_server()
+            self.__connect_after_start()
+        except (TimeoutError, STLError) as e:
+            LOG.error('Cannot connect to TRex')
+            LOG.error(traceback.format_exc())
+            logpath = '/tmp/trex.log'
+            if os.path.isfile(logpath):
+                # Wait for TRex to finish writing error message
+                last_size = 0
+                for _ in xrange(self.config.generic_retry_count):
+                    size = os.path.getsize(logpath)
+                    if size == last_size:
+                        # probably not writing anymore
+                        break
+                    last_size = size
+                    time.sleep(1)
+                with open(logpath, 'r') as f:
+                    message = f.read()
+            else:
+                message = e.message
+            raise TrafficGeneratorException(message)
+
     def __start_server(self):
         server = TRexTrafficServer()
         server.run_server(self.generator_config)
 
+    def __check_config(self):
+        server = TRexTrafficServer()
+        return server.check_config_updated(self.generator_config)
+
+    def __restart(self):
+        LOG.info("Restarting TRex ...")
+        self.__stop_server()
+        # Wait for server stopped
+        for _ in xrange(self.config.generic_retry_count):
+            time.sleep(1)
+            if not self.client.is_connected():
+                LOG.info("TRex is stopped...")
+                break
+        self.__start_local_server()
+
+    def __stop_server(self):
+        if self.generator_config.ip == '127.0.0.1':
+            ports = self.client.get_acquired_ports()
+            LOG.info('Release ports %s and stopping TRex...', ports)
+            try:
+                if ports:
+                    self.client.release(ports=ports)
+                self.client.server_shutdown()
+            except STLError as e:
+                LOG.warn('Unable to stop TRex. Error: %s', e)
+        else:
+            LOG.info('Using remote TRex. Unable to stop TRex')
+
     def resolve_arp(self):
         """Resolve all configured remote IP addresses.
 
diff --git a/nfvbench/traffic_server.py b/nfvbench/traffic_server.py
index df0a3be..d46a27d 100644
--- a/nfvbench/traffic_server.py
+++ b/nfvbench/traffic_server.py
@@ -57,9 +57,27 @@ class TRexTrafficServer(TrafficServer):
                          cwd=self.trex_dir)
         LOG.info('TRex server is running...')
 
+    def __load_config(self, filename):
+        result = {}
+        if os.path.exists(filename):
+            with open(filename, 'r') as stream:
+                try:
+                    result = yaml.load(stream)
+                except yaml.YAMLError as exc:
+                    print exc
+        return result
+
     def __save_config(self, generator_config, filename):
-        ifs = ",".join([repr(pci) for pci in generator_config.pcis])
+        result = self.__prepare_config(generator_config)
+        yaml.safe_load(result)
+        if os.path.exists(filename):
+            os.remove(filename)
+        with open(filename, 'w') as f:
+            f.write(result)
+        return filename
 
+    def __prepare_config(self, generator_config):
+        ifs = ",".join([repr(pci) for pci in generator_config.pcis])
         result = """# Config generated by NFVbench
         - port_limit : 2
           version    : 2
@@ -96,11 +114,13 @@ class TRexTrafficServer(TrafficServer):
             else:
                 LOG.info("Generator profile 'platform' sub-properties are set but not filled in \
                          config file. TRex will use default values.")
+        return result
 
-        yaml.safe_load(result)
-        if os.path.exists(filename):
-            os.remove(filename)
-        with open(filename, 'w') as f:
-            f.write(result)
-
-        return filename
+    def check_config_updated(self, generator_config):
+        existing_config = self.__load_config(filename='/etc/trex_cfg.yaml')
+        new_config = yaml.safe_load(self.__prepare_config(generator_config))
+        LOG.debug("Existing config: %s", existing_config)
+        LOG.debug("New config: %s", new_config)
+        if existing_config == new_config:
+            return False
+        return True