summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDanielMartinBuckley <daniel.m.buckley@intel.com>2018-09-10 19:02:10 +0100
committerDaniel Martin Buckley <daniel.m.buckley@intel.com>2018-10-02 10:36:39 +0000
commit7d5ff45a3b793f5e50b4a32b2438ab11fed6c899 (patch)
tree8e776a944986c39e89184d01a67335145f833eb5
parentee5d6413dbcbab40234e47ed3e74b85b704d6493 (diff)
NSB PROX NFVi Test does not stop after reaching expected precision
JIRA: YARDSTICK-1419 When using prox_binsearch algorithm, a binary search is performed, increasing lower bound when step was successful and decreasing upper bound when the step was a failure. This runs until the test_precision (as specified in the traffic profile) is reached. When the test precision is reached, the test is not completed until the runner duration is reached. As runner duration is usually high (e.g. 1800sec), the tests take much too long to execute. This makes it difficult to create test suites. Change-Id: I6cc503a09fb534a556c61c805e6df4786bb8cc4b Signed-off-by: Daniel Martin Buckley <daniel.m.buckley@intel.com>
-rw-r--r--yardstick/benchmark/runners/duration.py3
-rw-r--r--yardstick/benchmark/runners/proxduration.py3
-rw-r--r--yardstick/benchmark/scenarios/base.py3
-rw-r--r--yardstick/benchmark/scenarios/networking/vnf_generic.py3
-rw-r--r--yardstick/network_services/traffic_profile/base.py3
-rw-r--r--yardstick/network_services/traffic_profile/prox_binsearch.py3
-rw-r--r--yardstick/network_services/traffic_profile/prox_profile.py5
-rw-r--r--yardstick/network_services/vnf_generic/vnf/prox_helpers.py2
-rw-r--r--yardstick/tests/unit/benchmark/runner/test_duration.py18
-rw-r--r--yardstick/tests/unit/benchmark/runner/test_proxduration.py2
-rw-r--r--yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py4
-rw-r--r--yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py6
12 files changed, 36 insertions, 19 deletions
diff --git a/yardstick/benchmark/runners/duration.py b/yardstick/benchmark/runners/duration.py
index 14fd8bb47..55c3690fd 100644
--- a/yardstick/benchmark/runners/duration.py
+++ b/yardstick/benchmark/runners/duration.py
@@ -106,7 +106,8 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
sequence += 1
- if (errors and sla_action is None) or time.time() > timeout or aborted.is_set():
+ if ((errors and sla_action is None) or time.time() > timeout
+ or aborted.is_set() or benchmark.is_ended()):
LOG.info("Worker END")
break
diff --git a/yardstick/benchmark/runners/proxduration.py b/yardstick/benchmark/runners/proxduration.py
index 61a468fd3..e217904b9 100644
--- a/yardstick/benchmark/runners/proxduration.py
+++ b/yardstick/benchmark/runners/proxduration.py
@@ -112,7 +112,8 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
sequence += 1
- if (errors and sla_action is None) or time.time() > timeout or aborted.is_set():
+ if ((errors and sla_action is None) or time.time() > timeout
+ or aborted.is_set() or benchmark.is_ended()):
LOG.info("Worker END")
break
diff --git a/yardstick/benchmark/scenarios/base.py b/yardstick/benchmark/scenarios/base.py
index 90a87ac29..1737bb942 100644
--- a/yardstick/benchmark/scenarios/base.py
+++ b/yardstick/benchmark/scenarios/base.py
@@ -50,6 +50,9 @@ class Scenario(object):
def run(self, *args):
"""Entry point for scenario classes, called from runner worker"""
+ def is_ended(self):
+ return False
+
def teardown(self):
"""Default teardown implementation for Scenario classes"""
pass
diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py
index 6d68c5e64..20fff61ed 100644
--- a/yardstick/benchmark/scenarios/networking/vnf_generic.py
+++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py
@@ -63,6 +63,9 @@ class NetworkServiceTestCase(scenario_base.Scenario):
self.bin_path = get_nsb_option('bin_path', '')
self._mq_ids = []
+ def is_ended(self):
+ return self.traffic_profile is not None and self.traffic_profile.is_ended()
+
def _get_ip_flow_range(self, ip_start_range):
"""Retrieve a CIDR first and last viable IPs
diff --git a/yardstick/network_services/traffic_profile/base.py b/yardstick/network_services/traffic_profile/base.py
index 4fbceea9b..ea3f17874 100644
--- a/yardstick/network_services/traffic_profile/base.py
+++ b/yardstick/network_services/traffic_profile/base.py
@@ -97,6 +97,9 @@ class TrafficProfile(object):
self.params = tp_config
self.config = TrafficProfileConfig(tp_config)
+ def is_ended(self):
+ return False
+
def execute_traffic(self, traffic_generator, **kawrgs):
""" This methods defines the behavior of the traffic generator.
It will be called in a loop until the traffic generator exits.
diff --git a/yardstick/network_services/traffic_profile/prox_binsearch.py b/yardstick/network_services/traffic_profile/prox_binsearch.py
index 16a0411ec..5b3c9756e 100644
--- a/yardstick/network_services/traffic_profile/prox_binsearch.py
+++ b/yardstick/network_services/traffic_profile/prox_binsearch.py
@@ -66,6 +66,9 @@ class ProxBinSearchProfile(ProxProfile):
yield test_value
test_value = self.mid_point
+ def is_ended(self):
+ return self.done.is_set()
+
def run_test_with_pkt_size(self, traffic_gen, pkt_size, duration):
"""Run the test for a single packet size.
diff --git a/yardstick/network_services/traffic_profile/prox_profile.py b/yardstick/network_services/traffic_profile/prox_profile.py
index 343ef1da2..de4b3f9a0 100644
--- a/yardstick/network_services/traffic_profile/prox_profile.py
+++ b/yardstick/network_services/traffic_profile/prox_profile.py
@@ -16,6 +16,7 @@
from __future__ import absolute_import
import logging
+import multiprocessing
from yardstick.network_services.traffic_profile.base import TrafficProfile
from yardstick.network_services.vnf_generic.vnf.prox_helpers import ProxProfileHelper
@@ -56,7 +57,7 @@ class ProxProfile(TrafficProfile):
def __init__(self, tp_config):
super(ProxProfile, self).__init__(tp_config)
self.queue = None
- self.done = False
+ self.done = multiprocessing.Event()
self.results = []
# TODO: get init values from tp_config
@@ -116,7 +117,7 @@ class ProxProfile(TrafficProfile):
try:
pkt_size = next(self.pkt_size_iterator)
except StopIteration:
- self.done = True
+ self.done.set()
return
# Adjust packet size upwards if it's less than the minimum
diff --git a/yardstick/network_services/vnf_generic/vnf/prox_helpers.py b/yardstick/network_services/vnf_generic/vnf/prox_helpers.py
index 3241719e8..321c05779 100644
--- a/yardstick/network_services/vnf_generic/vnf/prox_helpers.py
+++ b/yardstick/network_services/vnf_generic/vnf/prox_helpers.py
@@ -984,7 +984,7 @@ class ProxResourceHelper(ClientResourceHelper):
def _run_traffic_once(self, traffic_profile):
traffic_profile.execute_traffic(self)
- if traffic_profile.done:
+ if traffic_profile.done.is_set():
self._queue.put({'done': True})
LOG.debug("tg_prox done")
self._terminated.value = 1
diff --git a/yardstick/tests/unit/benchmark/runner/test_duration.py b/yardstick/tests/unit/benchmark/runner/test_duration.py
index d4801ef2c..fa47e96bf 100644
--- a/yardstick/tests/unit/benchmark/runner/test_duration.py
+++ b/yardstick/tests/unit/benchmark/runner/test_duration.py
@@ -97,9 +97,9 @@ class DurationRunnerTest(unittest.TestCase):
multiprocessing.Event(), mock.Mock())
self._assert_defaults__worker_run_setup_and_teardown()
- self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 2)
- self.assertGreater(self.benchmark.my_method.call_count, 2)
- self.assertGreater(self.benchmark.post_run_wait_time.call_count, 2)
+ self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0)
+ self.assertGreater(self.benchmark.my_method.call_count, 0)
+ self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0)
def test__worker_process_called_without_cfg(self):
scenario_cfg = {'runner': {}}
@@ -140,9 +140,9 @@ class DurationRunnerTest(unittest.TestCase):
time.sleep(0.1)
self._assert_defaults__worker_run_setup_and_teardown()
- self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 2)
- self.assertGreater(self.benchmark.my_method.count, 103)
- self.assertGreater(self.benchmark.post_run_wait_time.call_count, 2)
+ self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0)
+ self.assertGreater(self.benchmark.my_method.count, 1)
+ self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0)
count = 101
while not output_queue.empty():
@@ -181,9 +181,9 @@ class DurationRunnerTest(unittest.TestCase):
time.sleep(0.1)
self._assert_defaults__worker_run_setup_and_teardown()
- self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 2)
- self.assertGreater(self.benchmark.my_method.count, 103)
- self.assertGreater(self.benchmark.post_run_wait_time.call_count, 2)
+ self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0)
+ self.assertGreater(self.benchmark.my_method.count, 1)
+ self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0)
count = 0
while not queue.empty():
diff --git a/yardstick/tests/unit/benchmark/runner/test_proxduration.py b/yardstick/tests/unit/benchmark/runner/test_proxduration.py
index 3299c5b05..056195fd3 100644
--- a/yardstick/tests/unit/benchmark/runner/test_proxduration.py
+++ b/yardstick/tests/unit/benchmark/runner/test_proxduration.py
@@ -97,7 +97,7 @@ class ProxDurationRunnerTest(unittest.TestCase):
{}, multiprocessing.Event(), mock.Mock())
self._assert_defaults__worker_run_setup_and_teardown()
- self.assertGreater(self.benchmark.my_method.call_count, 2)
+ self.assertGreater(self.benchmark.my_method.call_count, 0)
def test__worker_process_called_without_cfg(self):
scenario_cfg = {'runner': {}}
diff --git a/yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py b/yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py
index cf31cc27c..11bee03a4 100644
--- a/yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py
+++ b/yardstick/tests/unit/network_services/traffic_profile/test_prox_profile.py
@@ -100,13 +100,13 @@ class TestProxProfile(unittest.TestCase):
profile = ProxProfile(tp_config)
- self.assertFalse(profile.done)
+ self.assertFalse(profile.done.is_set())
for _ in packet_sizes:
with self.assertRaises(NotImplementedError):
profile.execute_traffic(traffic_generator)
self.assertIsNone(profile.execute_traffic(traffic_generator))
- self.assertTrue(profile.done)
+ self.assertTrue(profile.done.is_set())
def test_bounds_iterator(self):
tp_config = {
diff --git a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py
index 3b095647c..3d6ebb25b 100644
--- a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py
+++ b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py
@@ -1527,14 +1527,16 @@ class TestProxResourceHelper(unittest.TestCase):
def test_run_traffic(self):
setup_helper = mock.MagicMock()
helper = prox_helpers.ProxResourceHelper(setup_helper)
- traffic_profile = mock.MagicMock(**{"done": True})
+ traffic_profile = mock.MagicMock()
+ traffic_profile.done.is_set.return_value = True
helper.run_traffic(traffic_profile)
self.assertEqual(helper._terminated.value, 1)
def test__run_traffic_once(self):
setup_helper = mock.MagicMock()
helper = prox_helpers.ProxResourceHelper(setup_helper)
- traffic_profile = mock.MagicMock(**{"done": True})
+ traffic_profile = mock.MagicMock()
+ traffic_profile.done.is_set.return_value = True
helper._run_traffic_once(traffic_profile)
self.assertEqual(helper._terminated.value, 1)