Merge "NSB PROX NFVi Test does not stop after reaching expected precision"

12 files changed, 36 insertions, 19 deletions

diff --git a/yardstick/benchmark/runners/duration.py b/yardstick/benchmark/runners/duration.py
index 14fd8bb47..55c3690fd 100644
--- a/yardstick/benchmark/runners/duration.py
+++ b/yardstick/benchmark/runners/duration.py
@@ -106,7 +106,8 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
 
         sequence += 1
 
-        if (errors and sla_action is None) or time.time() > timeout or aborted.is_set():
+        if ((errors and sla_action is None) or time.time() > timeout
+                or aborted.is_set() or benchmark.is_ended()):
             LOG.info("Worker END")
             break
 
diff --git a/yardstick/benchmark/runners/proxduration.py b/yardstick/benchmark/runners/proxduration.py
index 61a468fd3..e217904b9 100644
--- a/yardstick/benchmark/runners/proxduration.py
+++ b/yardstick/benchmark/runners/proxduration.py
@@ -112,7 +112,8 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
 
         sequence += 1
 
-        if (errors and sla_action is None) or time.time() > timeout or aborted.is_set():
+        if ((errors and sla_action is None) or time.time() > timeout
+                or aborted.is_set() or benchmark.is_ended()):
             LOG.info("Worker END")
             break
 
diff --git a/yardstick/benchmark/scenarios/base.py b/yardstick/benchmark/scenarios/base.py
index 90a87ac29..1737bb942 100644
--- a/yardstick/benchmark/scenarios/base.py
+++ b/yardstick/benchmark/scenarios/base.py
@@ -50,6 +50,9 @@ class Scenario(object):
     def run(self, *args):
         """Entry point for scenario classes, called from runner worker"""
 
+    def is_ended(self):
+        return False
+
     def teardown(self):
         """Default teardown implementation for Scenario classes"""
         pass
diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py
index 6d68c5e64..20fff61ed 100644
--- a/yardstick/benchmark/scenarios/networking/vnf_generic.py
+++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py
@@ -63,6 +63,9 @@ class NetworkServiceTestCase(scenario_base.Scenario):
         self.bin_path = get_nsb_option('bin_path', '')
         self._mq_ids = []
 
+    def is_ended(self):
+        return self.traffic_profile is not None and self.traffic_profile.is_ended()
+
     def _get_ip_flow_range(self, ip_start_range):
         """Retrieve a CIDR first and last viable IPs
 
diff --git a/yardstick/network_services/traffic_profile/base.py b/yardstick/network_services/traffic_profile/base.py
index 4fbceea9b..ea3f17874 100644
--- a/yardstick/network_services/traffic_profile/base.py
+++ b/yardstick/network_services/traffic_profile/base.py
@@ -97,6 +97,9 @@ class TrafficProfile(object):
         self.params = tp_config
         self.config = TrafficProfileConfig(tp_config)
 
+    def is_ended(self):
+        return False
+
     def execute_traffic(self, traffic_generator, **kawrgs):
         """ This methods defines the behavior of the traffic generator.
         It will be called in a loop until the traffic generator exits.
diff --git a/yardstick/network_services/traffic_profile/prox_binsearch.py b/yardstick/network_services/traffic_profile/prox_binsearch.py
index 16a0411ec..5b3c9756e 100644
--- a/yardstick/network_services/traffic_profile/prox_binsearch.py
+++ b/yardstick/network_services/traffic_profile/prox_binsearch.py
@@ -66,6 +66,9 @@ class ProxBinSearchProfile(ProxProfile):
             yield test_value
             test_value = self.mid_point
 
+    def is_ended(self):
+        return self.done.is_set()
+
     def run_test_with_pkt_size(self, traffic_gen, pkt_size, duration):
         """Run the test for a single packet size.
 
diff --git a/yardstick/network_services/traffic_profile/prox_profile.py b/yardstick/network_services/traffic_profile/prox_profile.py
index 343ef1da2..de4b3f9a0 100644
--- a/yardstick/network_services/traffic_profile/prox_profile.py
+++ b/yardstick/network_services/traffic_profile/prox_profile.py
@@ -16,6 +16,7 @@
 from __future__ import absolute_import
 
 import logging
+import multiprocessing
 
 from yardstick.network_services.traffic_profile.base import TrafficProfile
 from yardstick.network_services.vnf_generic.vnf.prox_helpers import ProxProfileHelper
@@ -56,7 +57,7 @@ class ProxProfile(TrafficProfile):
     def __init__(self, tp_config):
         super(ProxProfile, self).__init__(tp_config)
         self.queue = None
-        self.done = False
+        self.done = multiprocessing.Event()
         self.results = []
 
         # TODO: get init values from tp_config
@@ -116,7 +117,7 @@ class ProxProfile(TrafficProfile):
         try:
             pkt_size = next(self.pkt_size_iterator)
         except StopIteration:
-            self.done = True
+            self.done.set()
             return
 
         # Adjust packet size upwards if it's less than the minimum
diff --git a/yardstick/network_services/vnf_generic/vnf/prox_helpers.py b/yardstick/network_services/vnf_generic/vnf/prox_helpers.py
index 3241719e8..321c05779 100644
--- a/yardstick/network_services/vnf_generic/vnf/prox_helpers.py
+++ b/yardstick/network_services/vnf_generic/vnf/prox_helpers.py
@@ -984,7 +984,7 @@ class ProxResourceHelper(ClientResourceHelper):
 
     def _run_traffic_once(self, traffic_profile):
         traffic_profile.execute_traffic(self)
-        if traffic_profile.done:
+        if traffic_profile.done.is_set():
             self._queue.put({'done': True})
             LOG.debug("tg_prox done")
             self._terminated.value = 1
diff --git a/yardstick/tests/unit/benchmark/runner/test_duration.py b/yardstick/tests/unit/benchmark/runner/test_duration.py
index d4801ef2c..fa47e96bf 100644
--- a/yardstick/tests/unit/benchmark/runner/test_duration.py
+++ b/yardstick/tests/unit/benchmark/runner/test_duration.py
@@ -97,9 +97,9 @@ class DurationRunnerTest(unittest.TestCase):
                                  multiprocessing.Event(), mock.Mock())
 
         self._assert_defaults__worker_run_setup_and_teardown()
-        self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 2)
-        self.assertGreater(self.benchmark.my_method.call_count, 2)
-        self.assertGreater(self.benchmark.post_run_wait_time.call_count, 2)
+        self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0)
+        self.assertGreater(self.benchmark.my_method.call_count, 0)
+        self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0)
 
     def test__worker_process_called_without_cfg(self):
         scenario_cfg = {'runner': {}}
@@ -140,9 +140,9 @@ class DurationRunnerTest(unittest.TestCase):
         time.sleep(0.1)
 
         self._assert_defaults__worker_run_setup_and_teardown()
-        self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 2)
-        self.assertGreater(self.benchmark.my_method.count, 103)
-        self.assertGreater(self.benchmark.post_run_wait_time.call_count, 2)
+        self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0)
+        self.assertGreater(self.benchmark.my_method.count, 1)
+        self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0)
 
         count = 101
         while not output_queue.empty():
@@ -181,9 +181,9 @@ class DurationRunnerTest(unittest.TestCase):
         time.sleep(0.1)
 
         self._assert_defaults__worker_run_setup_and_teardown()
-        self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 2)
-        self.assertGreater(self.benchmark.my_method.count, 103)
-        self.assertGreater(self.benchmark.post_run_wait_time.call_count, 2)
+        self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0)
+        self.assertGreater(self.benchmark.my_method.count, 1)
+        self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0)
 
         count = 0
         while not queue.empty():
diff --git a/yardstick/tests/unit/benchmark/runner/test_proxduration.py b/yardstick/tests/unit/benchmark/runner/test_proxduration.py
index 3299c5b05..056195fd3 100644
--- a/yardstick/tests/unit/benchmark/runner/test_proxduration.py
+++ b/yardstick/tests/unit/benchmark/runner/test_proxduration.py
@@ -97,7 +97,7 @@ class ProxDurationRunnerTest(unittest.TestCase):
             {}, multiprocessing.Event(), mock.Mock())
 
         self._assert_defaults__worker_run_setup_and_teardown()
-        self.assertGreater(self.benchmark.my_method.call_count, 2)
+        self.assertGreater(self.benchmark.my_method.call_count, 0)
 
     def test__worker_process_called_without_cfg(self):
         scenario_cfg = {'runner': {}}
diff --git a/yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py b/yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py
index cf31cc27c..11bee03a4 100644
--- a/yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py
+++ b/yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py
@@ -100,13 +100,13 @@ class TestProxProfile(unittest.TestCase):
 
         profile = ProxProfile(tp_config)
 
-        self.assertFalse(profile.done)
+        self.assertFalse(profile.done.is_set())
         for _ in packet_sizes:
             with self.assertRaises(NotImplementedError):
                 profile.execute_traffic(traffic_generator)
 
         self.assertIsNone(profile.execute_traffic(traffic_generator))
-        self.assertTrue(profile.done)
+        self.assertTrue(profile.done.is_set())
 
     def test_bounds_iterator(self):
         tp_config = {
diff --git a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py
index 3b095647c..3d6ebb25b 100644
--- a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py
+++ b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py
@@ -1527,14 +1527,16 @@ class TestProxResourceHelper(unittest.TestCase):
     def test_run_traffic(self):
         setup_helper = mock.MagicMock()
         helper = prox_helpers.ProxResourceHelper(setup_helper)
-        traffic_profile = mock.MagicMock(**{"done": True})
+        traffic_profile = mock.MagicMock()
+        traffic_profile.done.is_set.return_value = True
         helper.run_traffic(traffic_profile)
         self.assertEqual(helper._terminated.value, 1)
 
     def test__run_traffic_once(self):
         setup_helper = mock.MagicMock()
         helper = prox_helpers.ProxResourceHelper(setup_helper)
-        traffic_profile = mock.MagicMock(**{"done": True})
+        traffic_profile = mock.MagicMock()
+        traffic_profile.done.is_set.return_value = True
         helper._run_traffic_once(traffic_profile)
         self.assertEqual(helper._terminated.value, 1)