diff options
Diffstat (limited to 'yardstick/tests/unit/benchmark/runner')
| -rw-r--r-- | yardstick/tests/unit/benchmark/runner/__init__.py | 0 | ||||
| -rw-r--r-- | yardstick/tests/unit/benchmark/runner/test_arithmetic.py | 446 | ||||
| -rw-r--r-- | yardstick/tests/unit/benchmark/runner/test_base.py | 119 | ||||
| -rw-r--r-- | yardstick/tests/unit/benchmark/runner/test_duration.py | 315 | ||||
| -rw-r--r-- | yardstick/tests/unit/benchmark/runner/test_iteration.py | 45 | ||||
| -rw-r--r-- | yardstick/tests/unit/benchmark/runner/test_proxduration.py | 286 | ||||
| -rw-r--r-- | yardstick/tests/unit/benchmark/runner/test_search.py | 192 |
7 files changed, 1403 insertions, 0 deletions
diff --git a/yardstick/tests/unit/benchmark/runner/__init__.py b/yardstick/tests/unit/benchmark/runner/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/yardstick/tests/unit/benchmark/runner/__init__.py diff --git a/yardstick/tests/unit/benchmark/runner/test_arithmetic.py b/yardstick/tests/unit/benchmark/runner/test_arithmetic.py new file mode 100644 index 000000000..35d935cd5 --- /dev/null +++ b/yardstick/tests/unit/benchmark/runner/test_arithmetic.py @@ -0,0 +1,446 @@ +############################################################################## +# Copyright (c) 2018 Nokia and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import mock +import unittest +import multiprocessing +import os +import time + +from yardstick.benchmark.runners import arithmetic +from yardstick.common import exceptions as y_exc + + +class ArithmeticRunnerTest(unittest.TestCase): + class MyMethod(object): + SLA_VALIDATION_ERROR_SIDE_EFFECT = 1 + BROAD_EXCEPTION_SIDE_EFFECT = 2 + + def __init__(self, side_effect=0): + self.count = 101 + self.side_effect = side_effect + + def __call__(self, data): + self.count += 1 + data['my_key'] = self.count + if self.side_effect == self.SLA_VALIDATION_ERROR_SIDE_EFFECT: + raise y_exc.SLAValidationError(case_name='My Case', + error_msg='my error message') + elif self.side_effect == self.BROAD_EXCEPTION_SIDE_EFFECT: + raise y_exc.YardstickException + return self.count + + def setUp(self): + self.scenario_cfg = { + 'runner': { + 'interval': 0, + 'iter_type': 'nested_for_loops', + 'iterators': [ + { + 'name': 'stride', + 'start': 64, + 'stop': 128, + 'step': 64 + }, + { + 'name': 'size', + 'start': 500, + 'stop': 2000, + 'step': 500 + } + ] + }, + 'type': 'some_type' + } + + self.benchmark = mock.Mock() + self.benchmark_cls = mock.Mock(return_value=self.benchmark) + + def _assert_defaults__worker_process_run_setup_and_teardown(self): + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.teardown.assert_called_once() + + @mock.patch.object(os, 'getpid') + @mock.patch.object(multiprocessing, 'Process') + def test__run_benchmark_called_with(self, mock_multiprocessing_process, + mock_os_getpid): + mock_os_getpid.return_value = 101 + + runner = arithmetic.ArithmeticRunner({}) + benchmark_cls = mock.Mock() + runner._run_benchmark(benchmark_cls, 'my_method', self.scenario_cfg, + {}) + mock_multiprocessing_process.assert_called_once_with( + name='Arithmetic-some_type-101', + target=arithmetic._worker_process, + args=(runner.result_queue, benchmark_cls, 'my_method', + self.scenario_cfg, {}, runner.aborted, runner.output_queue)) + + @mock.patch.object(os, 'getpid') + def test__worker_process_runner_id(self, mock_os_getpid): + mock_os_getpid.return_value = 101 + + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self.assertEqual(self.scenario_cfg['runner']['runner_id'], 101) + + @mock.patch.object(time, 'sleep') + def test__worker_process_calls_nested_for_loops(self, mock_time_sleep): + self.scenario_cfg['runner']['interval'] = 99 + + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.benchmark.my_method.assert_has_calls([mock.call({})] * 8) + self.assertEqual(self.benchmark.my_method.call_count, 8) + mock_time_sleep.assert_has_calls([mock.call(99)] * 8) + self.assertEqual(mock_time_sleep.call_count, 8) + + @mock.patch.object(time, 'sleep') + def test__worker_process_calls_tuple_loops(self, mock_time_sleep): + self.scenario_cfg['runner']['interval'] = 99 + self.scenario_cfg['runner']['iter_type'] = 'tuple_loops' + + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.benchmark.my_method.assert_has_calls([mock.call({})] * 2) + self.assertEqual(self.benchmark.my_method.call_count, 2) + mock_time_sleep.assert_has_calls([mock.call(99)] * 2) + self.assertEqual(mock_time_sleep.call_count, 2) + + def test__worker_process_stored_options_nested_for_loops(self): + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 128, 'size': 2000}) + + def test__worker_process_stored_options_tuple_loops(self): + self.scenario_cfg['runner']['iter_type'] = 'tuple_loops' + + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 128, 'size': 1000}) + + def test__worker_process_aborted_set_early(self): + aborted = multiprocessing.Event() + aborted.set() + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + aborted, mock.Mock()) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.scenario_cfg['options'], {}) + self.benchmark.my_method.assert_not_called() + + def test__worker_process_output_queue_nested_for_loops(self): + self.benchmark.my_method = self.MyMethod() + + output_queue = multiprocessing.Queue() + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.01) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.count, 109) + result = [] + while not output_queue.empty(): + result.append(output_queue.get()) + self.assertListEqual(result, [102, 103, 104, 105, 106, 107, 108, 109]) + + def test__worker_process_output_queue_tuple_loops(self): + self.scenario_cfg['runner']['iter_type'] = 'tuple_loops' + self.benchmark.my_method = self.MyMethod() + + output_queue = multiprocessing.Queue() + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.01) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.count, 103) + result = [] + while not output_queue.empty(): + result.append(output_queue.get()) + self.assertListEqual(result, [102, 103]) + + def test__worker_process_queue_nested_for_loops(self): + self.benchmark.my_method = self.MyMethod() + + queue = multiprocessing.Queue() + timestamp = time.time() + arithmetic._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + time.sleep(0.01) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.count, 109) + count = 0 + while not queue.empty(): + count += 1 + result = queue.get() + self.assertEqual(result['errors'], '') + self.assertEqual(result['data'], {'my_key': count + 101}) + self.assertEqual(result['sequence'], count) + self.assertGreater(result['timestamp'], timestamp) + timestamp = result['timestamp'] + + def test__worker_process_queue_tuple_loops(self): + self.scenario_cfg['runner']['iter_type'] = 'tuple_loops' + self.benchmark.my_method = self.MyMethod() + + queue = multiprocessing.Queue() + timestamp = time.time() + arithmetic._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + time.sleep(0.01) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.count, 103) + count = 0 + while not queue.empty(): + count += 1 + result = queue.get() + self.assertEqual(result['errors'], '') + self.assertEqual(result['data'], {'my_key': count + 101}) + self.assertEqual(result['sequence'], count) + self.assertGreater(result['timestamp'], timestamp) + timestamp = result['timestamp'] + + def test__worker_process_except_sla_validation_error_no_sla_cfg(self): + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.call_count, 8) + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 128, 'size': 2000}) + + def test__worker_process_output_on_sla_validation_error_no_sla_cfg(self): + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.SLA_VALIDATION_ERROR_SIDE_EFFECT) + + queue = multiprocessing.Queue() + output_queue = multiprocessing.Queue() + timestamp = time.time() + arithmetic._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.01) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.count, 109) + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 128, 'size': 2000}) + count = 0 + while not queue.empty(): + count += 1 + result = queue.get() + self.assertEqual(result['errors'], '') + self.assertEqual(result['data'], {'my_key': count + 101}) + self.assertEqual(result['sequence'], count) + self.assertGreater(result['timestamp'], timestamp) + timestamp = result['timestamp'] + self.assertEqual(count, 8) + self.assertTrue(output_queue.empty()) + + def test__worker_process_except_sla_validation_error_sla_cfg_monitor(self): + self.scenario_cfg['sla'] = {'action': 'monitor'} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.call_count, 8) + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 128, 'size': 2000}) + + def test__worker_process_output_sla_validation_error_sla_cfg_monitor(self): + self.scenario_cfg['sla'] = {'action': 'monitor'} + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.SLA_VALIDATION_ERROR_SIDE_EFFECT) + + queue = multiprocessing.Queue() + output_queue = multiprocessing.Queue() + timestamp = time.time() + arithmetic._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.01) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.count, 109) + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 128, 'size': 2000}) + count = 0 + while not queue.empty(): + count += 1 + result = queue.get() + self.assertEqual(result['errors'], + ('My Case SLA validation failed. ' + 'Error: my error message',)) + self.assertEqual(result['data'], {'my_key': count + 101}) + self.assertEqual(result['sequence'], count) + self.assertGreater(result['timestamp'], timestamp) + timestamp = result['timestamp'] + self.assertEqual(count, 8) + self.assertTrue(output_queue.empty()) + + def test__worker_process_raise_sla_validation_error_sla_cfg_assert(self): + self.scenario_cfg['sla'] = {'action': 'assert'} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + + with self.assertRaises(y_exc.SLAValidationError): + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.my_method.assert_called_once() + self.benchmark.setup.assert_called_once() + self.benchmark.teardown.assert_not_called() + + def test__worker_process_output_sla_validation_error_sla_cfg_assert(self): + self.scenario_cfg['sla'] = {'action': 'assert'} + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.SLA_VALIDATION_ERROR_SIDE_EFFECT) + + queue = multiprocessing.Queue() + output_queue = multiprocessing.Queue() + with self.assertRaisesRegexp( + y_exc.SLAValidationError, + 'My Case SLA validation failed. Error: my error message'): + arithmetic._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.01) + + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.assertEqual(self.benchmark.my_method.count, 102) + self.benchmark.teardown.assert_not_called() + self.assertTrue(queue.empty()) + self.assertTrue(output_queue.empty()) + + def test__worker_process_broad_exception_no_sla_cfg_early_exit(self): + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.YardstickException) + + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.benchmark.my_method.assert_called_once() + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 64, 'size': 500}) + + def test__worker_process_output_on_broad_exception_no_sla_cfg(self): + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.BROAD_EXCEPTION_SIDE_EFFECT) + + queue = multiprocessing.Queue() + output_queue = multiprocessing.Queue() + timestamp = time.time() + arithmetic._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.01) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.count, 102) + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 64, 'size': 500}) + self.assertEqual(queue.qsize(), 1) + result = queue.get() + self.assertGreater(result['timestamp'], timestamp) + self.assertEqual(result['data'], {'my_key': 102}) + self.assertRegexpMatches( + result['errors'], + 'YardstickException: An unknown exception occurred.') + self.assertEqual(result['sequence'], 1) + self.assertTrue(output_queue.empty()) + + def test__worker_process_broad_exception_sla_cfg_not_none(self): + self.scenario_cfg['sla'] = {'action': 'some action'} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.YardstickException) + + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.call_count, 8) + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 128, 'size': 2000}) + + def test__worker_process_output_on_broad_exception_sla_cfg_not_none(self): + self.scenario_cfg['sla'] = {'action': 'some action'} + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.BROAD_EXCEPTION_SIDE_EFFECT) + + queue = multiprocessing.Queue() + output_queue = multiprocessing.Queue() + timestamp = time.time() + arithmetic._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.01) + + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.count, 109) + self.assertDictEqual(self.scenario_cfg['options'], + {'stride': 128, 'size': 2000}) + self.assertTrue(output_queue.empty()) + count = 0 + while not queue.empty(): + count += 1 + result = queue.get() + self.assertGreater(result['timestamp'], timestamp) + self.assertEqual(result['data'], {'my_key': count + 101}) + self.assertRegexpMatches( + result['errors'], + 'YardstickException: An unknown exception occurred.') + self.assertEqual(result['sequence'], count) + + def test__worker_process_benchmark_teardown_on_broad_exception(self): + self.benchmark.teardown = mock.Mock( + side_effect=y_exc.YardstickException) + + with self.assertRaises(SystemExit) as raised: + arithmetic._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + self.assertEqual(raised.exception.code, 1) + self._assert_defaults__worker_process_run_setup_and_teardown() + self.assertEqual(self.benchmark.my_method.call_count, 8) diff --git a/yardstick/tests/unit/benchmark/runner/test_base.py b/yardstick/tests/unit/benchmark/runner/test_base.py new file mode 100644 index 000000000..07d6f1843 --- /dev/null +++ b/yardstick/tests/unit/benchmark/runner/test_base.py @@ -0,0 +1,119 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import time + +import mock +import subprocess + +from yardstick.benchmark.runners import base as runner_base +from yardstick.benchmark.runners import iteration +from yardstick.tests.unit import base as ut_base + + +class ActionTestCase(ut_base.BaseUnitTestCase): + + def setUp(self): + self._mock_log = mock.patch.object(runner_base.log, 'error') + self.mock_log = self._mock_log.start() + self.addCleanup(self._stop_mocks) + + def _stop_mocks(self): + self._mock_log.stop() + + @mock.patch.object(subprocess, 'check_output') + def test__execute_shell_command(self, mock_subprocess): + mock_subprocess.side_effect = subprocess.CalledProcessError(-1, '') + self.assertEqual(runner_base._execute_shell_command("")[0], -1) + + @mock.patch.object(subprocess, 'check_output') + def test__single_action(self, mock_subprocess): + mock_subprocess.side_effect = subprocess.CalledProcessError(-1, '') + runner_base._single_action(0, 'echo', mock.Mock()) + + @mock.patch.object(subprocess, 'check_output') + def test__periodic_action(self, mock_subprocess): + mock_subprocess.side_effect = subprocess.CalledProcessError(-1, '') + runner_base._periodic_action(0, 'echo', mock.Mock()) + + +class ScenarioOutputTestCase(ut_base.BaseUnitTestCase): + + def setUp(self): + self.output_queue = mock.Mock() + self.scenario_output = runner_base.ScenarioOutput(self.output_queue, + sequence=1) + + @mock.patch.object(time, 'time') + def test_push(self, mock_time): + mock_time.return_value = 2 + data = {"value1": 1} + self.scenario_output.push(data) + self.output_queue.put.assert_called_once_with({'timestamp': 2, + 'sequence': 1, + 'data': data}, True, 10) + + def test_push_no_timestamp(self): + self.scenario_output["value1"] = 1 + self.scenario_output.push(None, False) + self.output_queue.put.assert_called_once_with({'sequence': 1, + 'value1': 1}, True, 10) + + +class RunnerTestCase(ut_base.BaseUnitTestCase): + + def setUp(self): + config = { + 'output_config': { + 'DEFAULT': { + 'dispatcher': 'file' + } + } + } + self.runner = iteration.IterationRunner(config) + + @mock.patch("yardstick.benchmark.runners.iteration.multiprocessing") + def test_get_output(self, *args): + self.runner.output_queue.put({'case': 'opnfv_yardstick_tc002'}) + self.runner.output_queue.put({'criteria': 'PASS'}) + + idle_result = { + 'case': 'opnfv_yardstick_tc002', + 'criteria': 'PASS' + } + + for _ in range(1000): + time.sleep(0.01) + if not self.runner.output_queue.empty(): + break + actual_result = self.runner.get_output() + self.assertEqual(idle_result, actual_result) + + @mock.patch("yardstick.benchmark.runners.iteration.multiprocessing") + def test_get_result(self, *args): + self.runner.result_queue.put({'case': 'opnfv_yardstick_tc002'}) + self.runner.result_queue.put({'criteria': 'PASS'}) + + idle_result = [ + {'case': 'opnfv_yardstick_tc002'}, + {'criteria': 'PASS'} + ] + + for _ in range(1000): + time.sleep(0.01) + if not self.runner.result_queue.empty(): + break + actual_result = self.runner.get_result() + self.assertEqual(idle_result, actual_result) + + def test__run_benchmark(self): + runner = runner_base.Runner(mock.Mock()) + + with self.assertRaises(NotImplementedError): + runner._run_benchmark(mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock()) diff --git a/yardstick/tests/unit/benchmark/runner/test_duration.py b/yardstick/tests/unit/benchmark/runner/test_duration.py new file mode 100644 index 000000000..fa47e96bf --- /dev/null +++ b/yardstick/tests/unit/benchmark/runner/test_duration.py @@ -0,0 +1,315 @@ +############################################################################## +# Copyright (c) 2018 Nokia and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import mock +import unittest +import multiprocessing +import os +import time + +from yardstick.benchmark.runners import duration +from yardstick.common import exceptions as y_exc + + +class DurationRunnerTest(unittest.TestCase): + class MyMethod(object): + SLA_VALIDATION_ERROR_SIDE_EFFECT = 1 + BROAD_EXCEPTION_SIDE_EFFECT = 2 + + def __init__(self, side_effect=0): + self.count = 101 + self.side_effect = side_effect + + def __call__(self, data): + self.count += 1 + data['my_key'] = self.count + if self.side_effect == self.SLA_VALIDATION_ERROR_SIDE_EFFECT: + raise y_exc.SLAValidationError(case_name='My Case', + error_msg='my error message') + elif self.side_effect == self.BROAD_EXCEPTION_SIDE_EFFECT: + raise y_exc.YardstickException + return self.count + + def setUp(self): + self.scenario_cfg = { + 'runner': {'interval': 0, "duration": 0}, + 'type': 'some_type' + } + + self.benchmark = mock.Mock() + self.benchmark_cls = mock.Mock(return_value=self.benchmark) + + def _assert_defaults__worker_run_setup_and_teardown(self): + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.teardown.assert_called_once() + + def _assert_defaults__worker_run_one_iteration(self): + self.benchmark.pre_run_wait_time.assert_called_once_with(0) + self.benchmark.my_method.assert_called_once_with({}) + self.benchmark.post_run_wait_time.assert_called_once_with(0) + + @mock.patch.object(os, 'getpid') + @mock.patch.object(multiprocessing, 'Process') + def test__run_benchmark_called_with(self, mock_multiprocessing_process, + mock_os_getpid): + mock_os_getpid.return_value = 101 + + runner = duration.DurationRunner({}) + benchmark_cls = mock.Mock() + runner._run_benchmark(benchmark_cls, 'my_method', self.scenario_cfg, + {}) + mock_multiprocessing_process.assert_called_once_with( + name='Duration-some_type-101', + target=duration._worker_process, + args=(runner.result_queue, benchmark_cls, 'my_method', + self.scenario_cfg, {}, runner.aborted, runner.output_queue)) + + @mock.patch.object(os, 'getpid') + def test__worker_process_runner_id(self, mock_os_getpid): + mock_os_getpid.return_value = 101 + + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self.assertEqual(self.scenario_cfg['runner']['runner_id'], 101) + + def test__worker_process_called_with_cfg(self): + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + self._assert_defaults__worker_run_one_iteration() + + def test__worker_process_called_with_cfg_loop(self): + self.scenario_cfg['runner']['duration'] = 0.01 + + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0) + self.assertGreater(self.benchmark.my_method.call_count, 0) + self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0) + + def test__worker_process_called_without_cfg(self): + scenario_cfg = {'runner': {}} + aborted = multiprocessing.Event() + aborted.set() + + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + scenario_cfg, {}, aborted, mock.Mock()) + + self.benchmark_cls.assert_called_once_with(scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.pre_run_wait_time.assert_called_once_with(1) + self.benchmark.my_method.assert_called_once_with({}) + self.benchmark.post_run_wait_time.assert_called_once_with(1) + self.benchmark.teardown.assert_called_once() + + def test__worker_process_output_queue(self): + self.benchmark.my_method = mock.Mock(return_value='my_result') + + output_queue = multiprocessing.Queue() + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.1) + + self._assert_defaults__worker_run_setup_and_teardown() + self._assert_defaults__worker_run_one_iteration() + self.assertEquals(output_queue.get(), 'my_result') + + def test__worker_process_output_queue_multiple_iterations(self): + self.scenario_cfg['runner']['duration'] = 0.01 + self.benchmark.my_method = self.MyMethod() + + output_queue = multiprocessing.Queue() + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), output_queue) + time.sleep(0.1) + + self._assert_defaults__worker_run_setup_and_teardown() + self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0) + self.assertGreater(self.benchmark.my_method.count, 1) + self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0) + + count = 101 + while not output_queue.empty(): + count += 1 + self.assertEquals(output_queue.get(), count) + + def test__worker_process_queue(self): + self.benchmark.my_method = self.MyMethod() + + queue = multiprocessing.Queue() + timestamp = time.time() + duration._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + time.sleep(0.1) + + self._assert_defaults__worker_run_setup_and_teardown() + self.benchmark.pre_run_wait_time.assert_called_once_with(0) + self.benchmark.post_run_wait_time.assert_called_once_with(0) + + result = queue.get() + self.assertGreater(result['timestamp'], timestamp) + self.assertEqual(result['errors'], '') + self.assertEqual(result['data'], {'my_key': 102}) + self.assertEqual(result['sequence'], 1) + + def test__worker_process_queue_multiple_iterations(self): + self.scenario_cfg['runner']['duration'] = 0.5 + self.benchmark.my_method = self.MyMethod() + + queue = multiprocessing.Queue() + timestamp = time.time() + duration._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + time.sleep(0.1) + + self._assert_defaults__worker_run_setup_and_teardown() + self.assertGreater(self.benchmark.pre_run_wait_time.call_count, 0) + self.assertGreater(self.benchmark.my_method.count, 1) + self.assertGreater(self.benchmark.post_run_wait_time.call_count, 0) + + count = 0 + while not queue.empty(): + count += 1 + result = queue.get() + self.assertGreater(result['timestamp'], timestamp) + self.assertEqual(result['errors'], '') + self.assertEqual(result['data'], {'my_key': count + 101}) + self.assertEqual(result['sequence'], count) + + def test__worker_process_except_sla_validation_error_no_sla_cfg(self): + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + self._assert_defaults__worker_run_one_iteration() + + def test__worker_process_except_sla_validation_error_sla_cfg_monitor(self): + self.scenario_cfg['sla'] = {'action': 'monitor'} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + self._assert_defaults__worker_run_one_iteration() + + def test__worker_process_raise_sla_validation_error_sla_cfg_default(self): + self.scenario_cfg['sla'] = {} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + + with self.assertRaises(y_exc.SLAValidationError): + duration._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.pre_run_wait_time.assert_called_once_with(0) + self.benchmark.my_method.assert_called_once_with({}) + + def test__worker_process_raise_sla_validation_error_sla_cfg_assert(self): + self.scenario_cfg['sla'] = {'action': 'assert'} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + + with self.assertRaises(y_exc.SLAValidationError): + duration._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.pre_run_wait_time.assert_called_once_with(0) + self.benchmark.my_method.assert_called_once_with({}) + + def test__worker_process_queue_on_sla_validation_error_monitor(self): + self.scenario_cfg['sla'] = {'action': 'monitor'} + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.SLA_VALIDATION_ERROR_SIDE_EFFECT) + + queue = multiprocessing.Queue() + timestamp = time.time() + duration._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + time.sleep(0.1) + + self._assert_defaults__worker_run_setup_and_teardown() + self.benchmark.pre_run_wait_time.assert_called_once_with(0) + self.benchmark.post_run_wait_time.assert_called_once_with(0) + + result = queue.get() + self.assertGreater(result['timestamp'], timestamp) + self.assertEqual(result['errors'], ('My Case SLA validation failed. ' + 'Error: my error message',)) + self.assertEqual(result['data'], {'my_key': 102}) + self.assertEqual(result['sequence'], 1) + + def test__worker_process_broad_exception(self): + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.YardstickException) + + duration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + self._assert_defaults__worker_run_one_iteration() + + def test__worker_process_queue_on_broad_exception(self): + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.BROAD_EXCEPTION_SIDE_EFFECT) + + queue = multiprocessing.Queue() + timestamp = time.time() + duration._worker_process(queue, self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + time.sleep(0.1) + + self._assert_defaults__worker_run_setup_and_teardown() + self.benchmark.pre_run_wait_time.assert_called_once_with(0) + self.benchmark.post_run_wait_time.assert_called_once_with(0) + + result = queue.get() + self.assertGreater(result['timestamp'], timestamp) + self.assertNotEqual(result['errors'], '') + self.assertEqual(result['data'], {'my_key': 102}) + self.assertEqual(result['sequence'], 1) + + def test__worker_process_benchmark_teardown_on_broad_exception(self): + self.benchmark.teardown = mock.Mock( + side_effect=y_exc.YardstickException) + + with self.assertRaises(SystemExit) as raised: + duration._worker_process(mock.Mock(), self.benchmark_cls, + 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + self.assertEqual(raised.exception.code, 1) + self._assert_defaults__worker_run_setup_and_teardown() + self._assert_defaults__worker_run_one_iteration() diff --git a/yardstick/tests/unit/benchmark/runner/test_iteration.py b/yardstick/tests/unit/benchmark/runner/test_iteration.py new file mode 100644 index 000000000..783b236f5 --- /dev/null +++ b/yardstick/tests/unit/benchmark/runner/test_iteration.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import mock +import unittest +import multiprocessing +from yardstick.benchmark.runners import iteration +from yardstick.common import exceptions as y_exc + + +class IterationRunnerTest(unittest.TestCase): + def setUp(self): + self.scenario_cfg = { + 'runner': {'interval': 0, "duration": 0}, + 'type': 'some_type' + } + + self.benchmark = mock.Mock() + self.benchmark_cls = mock.Mock(return_value=self.benchmark) + + def _assert_defaults__worker_run_setup_and_teardown(self): + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + + def _assert_defaults__worker_run_one_iteration(self): + self.benchmark.pre_run_wait_time.assert_called_once_with(0) + self.benchmark.my_method.assert_called_once_with({}) + + def test__worker_process_broad_exception(self): + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.YardstickException) + + with self.assertRaises(Exception): + iteration._worker_process(mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_one_iteration() + self._assert_defaults__worker_run_setup_and_teardown() diff --git a/yardstick/tests/unit/benchmark/runner/test_proxduration.py b/yardstick/tests/unit/benchmark/runner/test_proxduration.py new file mode 100644 index 000000000..056195fd3 --- /dev/null +++ b/yardstick/tests/unit/benchmark/runner/test_proxduration.py @@ -0,0 +1,286 @@ +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest +import multiprocessing +import os + +from yardstick.benchmark.runners import proxduration +from yardstick.common import constants +from yardstick.common import exceptions as y_exc + + +class ProxDurationRunnerTest(unittest.TestCase): + + class MyMethod(object): + SLA_VALIDATION_ERROR_SIDE_EFFECT = 1 + BROAD_EXCEPTION_SIDE_EFFECT = 2 + + def __init__(self, side_effect=0): + self.count = 101 + self.side_effect = side_effect + + def __call__(self, data): + self.count += 1 + data['my_key'] = self.count + if self.side_effect == self.SLA_VALIDATION_ERROR_SIDE_EFFECT: + raise y_exc.SLAValidationError(case_name='My Case', + error_msg='my error message') + elif self.side_effect == self.BROAD_EXCEPTION_SIDE_EFFECT: + raise y_exc.YardstickException + return self.count + + def setUp(self): + self.scenario_cfg = { + 'runner': {'interval': 0, "duration": 0}, + 'type': 'some_type' + } + + self.benchmark = mock.Mock() + self.benchmark_cls = mock.Mock(return_value=self.benchmark) + + def _assert_defaults__worker_run_setup_and_teardown(self): + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.teardown.assert_called_once() + + @mock.patch.object(os, 'getpid') + @mock.patch.object(multiprocessing, 'Process') + def test__run_benchmark_called_with(self, mock_multiprocessing_process, + mock_os_getpid): + mock_os_getpid.return_value = 101 + + runner = proxduration.ProxDurationRunner({}) + benchmark_cls = mock.Mock() + runner._run_benchmark(benchmark_cls, 'my_method', self.scenario_cfg, + {}) + mock_multiprocessing_process.assert_called_once_with( + name='ProxDuration-some_type-101', + target=proxduration._worker_process, + args=(runner.result_queue, benchmark_cls, 'my_method', + self.scenario_cfg, {}, runner.aborted, runner.output_queue)) + + @mock.patch.object(os, 'getpid') + def test__worker_process_runner_id(self, mock_os_getpid): + mock_os_getpid.return_value = 101 + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', self.scenario_cfg, + {}, multiprocessing.Event(), mock.Mock()) + + self.assertEqual(101, self.scenario_cfg['runner']['runner_id']) + + def test__worker_process_called_with_cfg(self): + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', self.scenario_cfg, + {}, multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + + def test__worker_process_called_with_cfg_loop(self): + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', self.scenario_cfg, + {}, multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + self.assertGreater(self.benchmark.my_method.call_count, 0) + + def test__worker_process_called_without_cfg(self): + scenario_cfg = {'runner': {}} + aborted = multiprocessing.Event() + aborted.set() + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', scenario_cfg, {}, + aborted, mock.Mock()) + + self.benchmark_cls.assert_called_once_with(scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.teardown.assert_called_once() + + def test__worker_process_output_queue(self): + self.benchmark.my_method = mock.Mock(return_value='my_result') + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + output_queue = mock.Mock() + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', self.scenario_cfg, + {}, multiprocessing.Event(), output_queue) + + self._assert_defaults__worker_run_setup_and_teardown() + output_queue.put.assert_has_calls( + [mock.call('my_result', True, constants.QUEUE_PUT_TIMEOUT)]) + + def test__worker_process_output_queue_multiple_iterations(self): + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + self.benchmark.my_method = self.MyMethod() + output_queue = mock.Mock() + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', self.scenario_cfg, + {}, multiprocessing.Event(), output_queue) + + self._assert_defaults__worker_run_setup_and_teardown() + for idx in range(102, 101 + len(output_queue.method_calls)): + output_queue.put.assert_has_calls( + [mock.call(idx, True, constants.QUEUE_PUT_TIMEOUT)]) + + def test__worker_process_queue(self): + self.benchmark.my_method = self.MyMethod() + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + queue = mock.Mock() + proxduration._worker_process( + queue, self.benchmark_cls, 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + benchmark_output = {'timestamp': mock.ANY, + 'sequence': 1, + 'data': {'my_key': 102}, + 'errors': ''} + queue.put.assert_has_calls( + [mock.call(benchmark_output, True, constants.QUEUE_PUT_TIMEOUT)]) + + def test__worker_process_queue_multiple_iterations(self): + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + self.benchmark.my_method = self.MyMethod() + queue = mock.Mock() + proxduration._worker_process( + queue, self.benchmark_cls, 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + for idx in range(102, 101 + len(queue.method_calls)): + benchmark_output = {'timestamp': mock.ANY, + 'sequence': idx - 101, + 'data': {'my_key': idx}, + 'errors': ''} + queue.put.assert_has_calls( + [mock.call(benchmark_output, True, + constants.QUEUE_PUT_TIMEOUT)]) + + def test__worker_process_except_sla_validation_error_no_sla_cfg(self): + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', self.scenario_cfg, + {}, multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + + @mock.patch.object(proxduration.LOG, 'warning') + def test__worker_process_except_sla_validation_error_sla_cfg_monitor( + self, *args): + self.scenario_cfg['sla'] = {'action': 'monitor'} + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', self.scenario_cfg, + {}, multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + + def test__worker_process_raise_sla_validation_error_sla_cfg_default(self): + self.scenario_cfg['sla'] = {} + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + with self.assertRaises(y_exc.SLAValidationError): + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, multiprocessing.Event(), mock.Mock()) + + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.my_method.assert_called_once_with({}) + + def test__worker_process_raise_sla_validation_error_sla_cfg_assert(self): + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + self.scenario_cfg['sla'] = {'action': 'assert'} + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.SLAValidationError) + + with self.assertRaises(y_exc.SLAValidationError): + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, multiprocessing.Event(), mock.Mock()) + + self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {}) + self.benchmark.setup.assert_called_once() + self.benchmark.my_method.assert_called_once_with({}) + + @mock.patch.object(proxduration.LOG, 'warning') + def test__worker_process_queue_on_sla_validation_error_monitor( + self, *args): + self.scenario_cfg['sla'] = {'action': 'monitor'} + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.SLA_VALIDATION_ERROR_SIDE_EFFECT) + queue = mock.Mock() + proxduration._worker_process( + queue, self.benchmark_cls, 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + benchmark_output = {'timestamp': mock.ANY, + 'sequence': 1, + 'data': {'my_key': 102}, + 'errors': ('My Case SLA validation failed. ' + 'Error: my error message', )} + queue.put.assert_has_calls( + [mock.call(benchmark_output, True, constants.QUEUE_PUT_TIMEOUT)]) + + @mock.patch.object(proxduration.LOG, 'exception') + def test__worker_process_broad_exception(self, *args): + self.benchmark.my_method = mock.Mock( + side_effect=y_exc.YardstickException) + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, multiprocessing.Event(), mock.Mock()) + + self._assert_defaults__worker_run_setup_and_teardown() + + @mock.patch.object(proxduration.LOG, 'exception') + def test__worker_process_queue_on_broad_exception(self, *args): + self.benchmark.my_method = self.MyMethod( + side_effect=self.MyMethod.BROAD_EXCEPTION_SIDE_EFFECT) + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + queue = mock.Mock() + proxduration._worker_process( + queue, self.benchmark_cls, 'my_method', self.scenario_cfg, {}, + multiprocessing.Event(), mock.Mock()) + + benchmark_output = {'timestamp': mock.ANY, + 'sequence': 1, + 'data': {'my_key': 102}, + 'errors': mock.ANY} + queue.put.assert_has_calls( + [mock.call(benchmark_output, True, constants.QUEUE_PUT_TIMEOUT)]) + + @mock.patch.object(proxduration.LOG, 'exception') + def test__worker_process_benchmark_teardown_on_broad_exception( + self, *args): + self.benchmark.teardown = mock.Mock( + side_effect=y_exc.YardstickException) + self.scenario_cfg["runner"] = {"sampled": True, "duration": 0.1} + + with self.assertRaises(SystemExit) as raised: + proxduration._worker_process( + mock.Mock(), self.benchmark_cls, 'my_method', + self.scenario_cfg, {}, multiprocessing.Event(), mock.Mock()) + self.assertEqual(1, raised.exception.code) + self._assert_defaults__worker_run_setup_and_teardown() diff --git a/yardstick/tests/unit/benchmark/runner/test_search.py b/yardstick/tests/unit/benchmark/runner/test_search.py new file mode 100644 index 000000000..d5d1b8ded --- /dev/null +++ b/yardstick/tests/unit/benchmark/runner/test_search.py @@ -0,0 +1,192 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import mock +import unittest + +from yardstick.benchmark.runners.search import SearchRunner +from yardstick.benchmark.runners.search import SearchRunnerHelper +from yardstick.common import exceptions as y_exc + + +class TestSearchRunnerHelper(unittest.TestCase): + + def test___call__(self): + scenario_cfg = { + 'runner': {}, + } + + benchmark = mock.Mock() + method = getattr(benchmark(), 'my_method') + helper = SearchRunnerHelper( + benchmark, 'my_method', scenario_cfg, {}, mock.Mock()) + + with helper.get_benchmark_instance(): + helper() + + method.assert_called_once() + + def test___call___error(self): + scenario_cfg = { + 'runner': {}, + } + + helper = SearchRunnerHelper( + mock.Mock(), 'my_method', scenario_cfg, {}, mock.Mock()) + + with self.assertRaises(RuntimeError): + helper() + + @mock.patch.object(time, 'sleep') + @mock.patch.object(time, 'time') + def test_is_not_done(self, mock_time, *args): + scenario_cfg = { + 'runner': {}, + } + + mock_time.side_effect = range(1000) + + helper = SearchRunnerHelper( + mock.Mock(), 'my_method', scenario_cfg, {}, mock.Mock()) + + index = -1 + for index in helper.is_not_done(): + if index >= 10: + break + + self.assertGreaterEqual(index, 10) + + @mock.patch.object(time, 'sleep') + def test_is_not_done_immediate_stop(self, *args): + scenario_cfg = { + 'runner': { + 'run_step': '', + }, + } + + helper = SearchRunnerHelper( + mock.Mock(), 'my_method', scenario_cfg, {}, mock.Mock()) + + index = -1 + for index in helper.is_not_done(): + if index >= 10: + break + + self.assertEqual(index, -1) + + +class TestSearchRunner(unittest.TestCase): + + def test__worker_run_once(self): + def update(*args): + args[-1].update(data) + + data = { + 'key1': { + 'inner1': 'value1', + 'done': 0, + }, + 'key2': { + 'done': None, + }, + } + + runner = SearchRunner({}) + runner.worker_helper = mock.Mock(side_effect=update) + + self.assertFalse(runner._worker_run_once('sequence 1')) + + def test__worker_run_once_done(self): + def update(*args): + args[-1].update(data) + + data = { + 'key1': { + 'inner1': 'value1', + 'done': 0, + }, + 'key2': { + 'done': None, + }, + 'key3': { + 'done': True, + }, + 'key4': [], + 'key5': 'value5', + } + + runner = SearchRunner({}) + runner.worker_helper = mock.Mock(side_effect=update) + + self.assertTrue(runner._worker_run_once('sequence 1')) + + def test__worker_run_once_assertion_error_assert(self): + runner = SearchRunner({}) + runner.sla_action = 'assert' + runner.worker_helper = mock.Mock(side_effect=y_exc.SLAValidationError) + + with self.assertRaises(y_exc.SLAValidationError): + runner._worker_run_once('sequence 1') + + def test__worker_run_once_assertion_error_monitor(self): + runner = SearchRunner({}) + runner.sla_action = 'monitor' + runner.worker_helper = mock.Mock(side_effect=y_exc.SLAValidationError) + + self.assertFalse(runner._worker_run_once('sequence 1')) + + def test__worker_run_once_non_assertion_error_none(self): + runner = SearchRunner({}) + runner.worker_helper = mock.Mock(side_effect=RuntimeError) + + self.assertTrue(runner._worker_run_once('sequence 1')) + + def test__worker_run_once_non_assertion_error(self): + runner = SearchRunner({}) + runner.sla_action = 'monitor' + runner.worker_helper = mock.Mock(side_effect=RuntimeError) + + self.assertFalse(runner._worker_run_once('sequence 1')) + + def test__worker_run(self): + scenario_cfg = { + 'runner': {'interval': 0, 'timeout': 1}, + } + + runner = SearchRunner({}) + runner._worker_run_once = mock.Mock(side_effect=[0, 0, 1]) + + runner._worker_run(mock.Mock(), 'my_method', scenario_cfg, {}) + + def test__worker_run_immediate_stop(self): + scenario_cfg = { + 'runner': { + 'run_step': '', + }, + } + + runner = SearchRunner({}) + runner._worker_run(mock.Mock(), 'my_method', scenario_cfg, {}) + + @mock.patch('yardstick.benchmark.runners.search.multiprocessing') + def test__run_benchmark(self, mock_multi_process): + scenario_cfg = { + 'runner': {}, + } + + runner = SearchRunner({}) + runner._run_benchmark(mock.Mock(), 'my_method', scenario_cfg, {}) + mock_multi_process.Process.assert_called_once() |