NFVBENCH-56 Multi-chaining traffic starts may be too early for some runs

Change-Id: I332a53e3dd3e14e9cba4ad9f57bdfd094ffa4d3a
Signed-off-by: Kerim Gokarslan <kgokarsl@cisco.com>

3 files changed, 59 insertions, 28 deletions

diff --git a/nfvbench/traffic_client.py b/nfvbench/traffic_client.py
index bdcc027..056075a 100755
--- a/nfvbench/traffic_client.py
+++ b/nfvbench/traffic_client.py
@@ -453,51 +453,49 @@ class TrafficClient(object):
         Ensure traffic generator receives packets it has transmitted.
         This ensures end to end connectivity and also waits until VMs are ready to forward packets.
 
-        At this point all VMs are in active state, but forwarding does not have to work.
-        Small amount of traffic is sent to every chain. Then total of sent and received packets
-        is compared. If ratio between received and transmitted packets is higher than (N-1)/N,
-        N being number of chains, traffic flows through every chain and real measurements can be
-        performed.
+        VMs that are started and in active state may not pass traffic yet. It is imperative to make
+        sure that all VMs are passing traffic in both directions before starting any benchmarking.
+        To verify this, we need to send at a low frequency bi-directional packets and make sure
+        that we receive all packets back from all VMs. The number of flows is equal to 2 times
+        the number of chains (1 per direction) and we need to make sure we receive packets coming
+        from exactly 2 x chain count different source MAC addresses.
 
         Example:
             PVP chain (1 VM per chain)
             N = 10 (number of chains)
-            threshold = (N-1)/N = 9/10 = 0.9 (acceptable ratio ensuring working conditions)
-            if total_received/total_sent > 0.9, traffic is flowing to more than 9 VMs meaning
-            all 10 VMs are in operational state.
+            Flow count = 20 (number of flows)
+            If the number of unique source MAC addresses from received packets is 20 then
+            all 10 VMs 10 VMs are in operational state.
         """
         LOG.info('Starting traffic generator to ensure end-to-end connectivity')
-        rate_pps = {'rate_pps': str(self.config.service_chain_count * 100)}
+        rate_pps = {'rate_pps': str(self.config.service_chain_count * 1)}
         self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False)
 
         # ensures enough traffic is coming back
-        threshold = (self.config.service_chain_count - 1) / float(self.config.service_chain_count)
         retry_count = (self.config.check_traffic_time_sec +
                        self.config.generic_poll_sec - 1) / self.config.generic_poll_sec
+        mac_addresses = set()
+        ln = 0
         for it in xrange(retry_count):
             self.gen.clear_stats()
             self.gen.start_traffic()
+            self.gen.start_capture()
             LOG.info('Waiting for packets to be received back... (%d / %d)', it + 1, retry_count)
             if not self.skip_sleep:
                 time.sleep(self.config.generic_poll_sec)
             self.gen.stop_traffic()
-            stats = self.gen.get_stats()
-
-            # compute total sent and received traffic on both ports
-            total_rx = 0
-            total_tx = 0
-            for port in self.PORTS:
-                total_rx += float(stats[port]['rx'].get('total_pkts', 0))
-                total_tx += float(stats[port]['tx'].get('total_pkts', 0))
-
-            # how much of traffic came back
-            ratio = total_rx / total_tx if total_tx else 0
-
-            if ratio > threshold:
-                self.gen.clear_stats()
-                self.gen.clear_streamblock()
-                LOG.info('End-to-end connectivity ensured')
-                return
+            self.gen.fetch_capture_packets()
+            self.gen.stop_capture()
+
+            for packet in self.gen.packet_list:
+                mac_addresses.add(packet['binary'][6:12])
+                if ln != len(mac_addresses):
+                    ln = len(mac_addresses)
+                    LOG.info('Flows passing traffic %d / %d', ln,
+                             self.config.service_chain_count * 2)
+                if len(mac_addresses) == self.config.service_chain_count * 2:
+                    LOG.info('End-to-end connectivity ensured')
+                    return
 
             if not self.skip_sleep:
                 time.sleep(self.config.generic_poll_sec)
@@ -839,7 +837,6 @@ class TrafficClient(object):
         for direction in ['orig', 'tx', 'rx']:
             total[direction] = {}
             for unit in ['rate_percent', 'rate_bps', 'rate_pps']:
-
                 total[direction][unit] = sum([float(x[direction][unit]) for x in r.values()])
 
         r['direction-total'] = total
diff --git a/nfvbench/traffic_gen/dummy.py b/nfvbench/traffic_gen/dummy.py
index 6f57f4d..788a53f 100644
--- a/nfvbench/traffic_gen/dummy.py
+++ b/nfvbench/traffic_gen/dummy.py
@@ -31,6 +31,11 @@ class DummyTG(AbstractTrafficGenerator):
         self.duration_sec = self.config.duration_sec
         self.intf_speed = config.generator_config.intf_speed
         self.set_response_curve()
+        self.packet_list = [{
+            "binary": "01234567890123456789"
+        }, {
+            "binary": "98765432109876543210"
+        }]
 
     def get_version(self):
         return "0.1"
@@ -164,9 +169,18 @@ class DummyTG(AbstractTrafficGenerator):
     def start_traffic(self):
         pass
 
+    def fetch_capture_packets(self):
+        pass
+
     def stop_traffic(self):
         pass
 
+    def start_capture(self):
+        pass
+
+    def stop_capture(self):
+        pass
+
     def cleanup(self):
         pass
 
diff --git a/nfvbench/traffic_gen/trex.py b/nfvbench/traffic_gen/trex.py
index 207fd52..4c9f492 100644
--- a/nfvbench/traffic_gen/trex.py
+++ b/nfvbench/traffic_gen/trex.py
@@ -66,6 +66,8 @@ class TRex(AbstractTrafficGenerator):
         self.streamblock = defaultdict(list)
         self.rates = []
         self.arps = {}
+        self.capture_id = None
+        self.packet_list = []
 
     def get_version(self):
         return self.client.get_server_version()
@@ -455,6 +457,24 @@ class TRex(AbstractTrafficGenerator):
     def stop_traffic(self):
         self.client.stop(ports=self.port_handle)
 
+    def start_capture(self):
+        if self.capture_id:
+            self.stop_capture()
+        self.client.set_service_mode(ports=self.port_handle)
+        self.capture_id = self.client.start_capture(rx_ports=self.port_handle)
+
+    def fetch_capture_packets(self):
+        if self.capture_id:
+            self.packet_list = []
+            self.client.fetch_capture_packets(capture_id=self.capture_id['id'],
+                                              output=self.packet_list)
+
+    def stop_capture(self):
+        if self.capture_id:
+            self.client.stop_capture(capture_id=self.capture_id['id'])
+            self.capture_id = None
+            self.client.set_service_mode(ports=self.port_handle, enabled=False)
+
     def cleanup(self):
         if self.client:
             try: