aboutsummaryrefslogtreecommitdiffstats
path: root/tests/unit/benchmark
diff options
context:
space:
mode:
Diffstat (limited to 'tests/unit/benchmark')
-rw-r--r--tests/unit/benchmark/runner/test_base.py7
-rw-r--r--tests/unit/benchmark/runner/test_search.py263
-rw-r--r--tests/unit/benchmark/scenarios/lib/test_get_numa_info.py6
-rw-r--r--tests/unit/benchmark/scenarios/networking/test_vsperf_dpdk.py236
4 files changed, 509 insertions, 3 deletions
diff --git a/tests/unit/benchmark/runner/test_base.py b/tests/unit/benchmark/runner/test_base.py
index 0313ef843..956762c40 100644
--- a/tests/unit/benchmark/runner/test_base.py
+++ b/tests/unit/benchmark/runner/test_base.py
@@ -17,6 +17,7 @@ import time
from mock import mock
+from yardstick.benchmark.runners.base import Runner
from yardstick.benchmark.runners.iteration import IterationRunner
@@ -40,6 +41,12 @@ class RunnerTestCase(unittest.TestCase):
actual_result = runner.get_output()
self.assertEqual(idle_result, actual_result)
+ def test__run_benchmark(self):
+ runner = Runner(mock.Mock())
+
+ with self.assertRaises(NotImplementedError):
+ runner._run_benchmark(mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock())
+
def main():
unittest.main()
diff --git a/tests/unit/benchmark/runner/test_search.py b/tests/unit/benchmark/runner/test_search.py
new file mode 100644
index 000000000..9cfe6e154
--- /dev/null
+++ b/tests/unit/benchmark/runner/test_search.py
@@ -0,0 +1,263 @@
+# Copyright (c) 2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+import unittest
+from contextlib import contextmanager
+
+import mock
+
+STL_MOCKS = {
+ 'stl': mock.MagicMock(),
+ 'stl.trex_stl_lib': mock.MagicMock(),
+ 'stl.trex_stl_lib.base64': mock.MagicMock(),
+ 'stl.trex_stl_lib.binascii': mock.MagicMock(),
+ 'stl.trex_stl_lib.collections': mock.MagicMock(),
+ 'stl.trex_stl_lib.copy': mock.MagicMock(),
+ 'stl.trex_stl_lib.datetime': mock.MagicMock(),
+ 'stl.trex_stl_lib.functools': mock.MagicMock(),
+ 'stl.trex_stl_lib.imp': mock.MagicMock(),
+ 'stl.trex_stl_lib.inspect': mock.MagicMock(),
+ 'stl.trex_stl_lib.json': mock.MagicMock(),
+ 'stl.trex_stl_lib.linecache': mock.MagicMock(),
+ 'stl.trex_stl_lib.math': mock.MagicMock(),
+ 'stl.trex_stl_lib.os': mock.MagicMock(),
+ 'stl.trex_stl_lib.platform': mock.MagicMock(),
+ 'stl.trex_stl_lib.pprint': mock.MagicMock(),
+ 'stl.trex_stl_lib.random': mock.MagicMock(),
+ 'stl.trex_stl_lib.re': mock.MagicMock(),
+ 'stl.trex_stl_lib.scapy': mock.MagicMock(),
+ 'stl.trex_stl_lib.socket': mock.MagicMock(),
+ 'stl.trex_stl_lib.string': mock.MagicMock(),
+ 'stl.trex_stl_lib.struct': mock.MagicMock(),
+ 'stl.trex_stl_lib.sys': mock.MagicMock(),
+ 'stl.trex_stl_lib.threading': mock.MagicMock(),
+ 'stl.trex_stl_lib.time': mock.MagicMock(),
+ 'stl.trex_stl_lib.traceback': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_async_client': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_client': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_exceptions': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_ext': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_jsonrpc_client': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_packet_builder_interface': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_packet_builder_scapy': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_port': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_stats': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_streams': mock.MagicMock(),
+ 'stl.trex_stl_lib.trex_stl_types': mock.MagicMock(),
+ 'stl.trex_stl_lib.types': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.argparse': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.collections': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.common': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.json': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.os': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.parsing_opts': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.pwd': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.random': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.re': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.string': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.sys': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.text_opts': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.text_tables': mock.MagicMock(),
+ 'stl.trex_stl_lib.utils.texttable': mock.MagicMock(),
+ 'stl.trex_stl_lib.warnings': mock.MagicMock(),
+ 'stl.trex_stl_lib.yaml': mock.MagicMock(),
+ 'stl.trex_stl_lib.zlib': mock.MagicMock(),
+ 'stl.trex_stl_lib.zmq': mock.MagicMock(),
+}
+
+STLClient = mock.MagicMock()
+stl_patch = mock.patch.dict("sys.modules", STL_MOCKS)
+stl_patch.start()
+
+if stl_patch:
+ from yardstick.benchmark.runners.search import SearchRunner
+ from yardstick.benchmark.runners.search import SearchRunnerHelper
+
+
+class TestSearchRunnerHelper(unittest.TestCase):
+
+ def test___call__(self):
+ cls = mock.MagicMock()
+ aborted = mock.MagicMock()
+ scenario_cfg = {
+ 'runner': {},
+ }
+
+ benchmark = cls()
+ method = getattr(benchmark, 'my_method')
+ helper = SearchRunnerHelper(cls, 'my_method', scenario_cfg, {}, aborted)
+
+ with helper.get_benchmark_instance():
+ helper()
+
+ self.assertEqual(method.call_count, 1)
+
+ def test___call___error(self):
+ cls = mock.MagicMock()
+ aborted = mock.MagicMock()
+ scenario_cfg = {
+ 'runner': {},
+ }
+
+ helper = SearchRunnerHelper(cls, 'my_method', scenario_cfg, {}, aborted)
+
+ with self.assertRaises(RuntimeError):
+ helper()
+
+ @mock.patch('yardstick.benchmark.runners.search.time')
+ def test_is_not_done(self, mock_time):
+ cls = mock.MagicMock()
+ aborted = mock.MagicMock()
+ scenario_cfg = {
+ 'runner': {},
+ }
+
+ mock_time.time.side_effect = range(1000)
+
+ helper = SearchRunnerHelper(cls, 'my_method', scenario_cfg, {}, aborted)
+
+ index = -1
+ for index in helper.is_not_done():
+ if index >= 10:
+ break
+
+ self.assertGreaterEqual(index, 10)
+
+ @mock.patch('yardstick.benchmark.runners.search.time')
+ def test_is_not_done_immediate_stop(self, mock_time):
+ cls = mock.MagicMock()
+ aborted = mock.MagicMock()
+ scenario_cfg = {
+ 'runner': {
+ 'run_step': '',
+ },
+ }
+
+ helper = SearchRunnerHelper(cls, 'my_method', scenario_cfg, {}, aborted)
+
+ index = -1
+ for index in helper.is_not_done():
+ if index >= 10:
+ break
+
+ self.assertEqual(index, -1)
+
+class TestSearchRunner(unittest.TestCase):
+
+ def test__worker_run_once(self):
+ def update(*args):
+ args[-1].update(data)
+
+ data = {
+ 'key1': {
+ 'inner1': 'value1',
+ 'done': 0,
+ },
+ 'key2': {
+ 'done': None,
+ },
+ }
+
+ runner = SearchRunner({})
+ runner.worker_helper = mock.MagicMock(side_effect=update)
+
+ self.assertFalse(runner._worker_run_once('sequence 1'))
+
+ def test__worker_run_once_done(self):
+ def update(*args):
+ args[-1].update(data)
+
+ data = {
+ 'key1': {
+ 'inner1': 'value1',
+ 'done': 0,
+ },
+ 'key2': {
+ 'done': None,
+ },
+ 'key3': {
+ 'done': True,
+ },
+ 'key4': [],
+ 'key5': 'value5',
+ }
+
+ runner = SearchRunner({})
+ runner.worker_helper = mock.MagicMock(side_effect=update)
+
+ self.assertTrue(runner._worker_run_once('sequence 1'))
+
+ def test__worker_run_once_assertion_error_assert(self):
+ runner = SearchRunner({})
+ runner.sla_action = 'assert'
+ runner.worker_helper = mock.MagicMock(side_effect=AssertionError)
+
+ with self.assertRaises(AssertionError):
+ runner._worker_run_once('sequence 1')
+
+ def test__worker_run_once_assertion_error_monitor(self):
+ runner = SearchRunner({})
+ runner.sla_action = 'monitor'
+ runner.worker_helper = mock.MagicMock(side_effect=AssertionError)
+
+ self.assertFalse(runner._worker_run_once('sequence 1'))
+
+ def test__worker_run_once_non_assertion_error_none(self):
+ runner = SearchRunner({})
+ runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
+
+ self.assertTrue(runner._worker_run_once('sequence 1'))
+
+ def test__worker_run_once_non_assertion_error(self):
+ runner = SearchRunner({})
+ runner.sla_action = 'monitor'
+ runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
+
+ self.assertFalse(runner._worker_run_once('sequence 1'))
+
+ def test__worker_run(self):
+ cls = mock.MagicMock()
+ scenario_cfg = {
+ 'runner': {'interval': 0, 'timeout': 1},
+ }
+
+ runner = SearchRunner({})
+ runner._worker_run_once = mock.MagicMock(side_effect=[0, 0, 1])
+
+ runner._worker_run(cls, 'my_method', scenario_cfg, {})
+
+ def test__worker_run_immediate_stop(self):
+ cls = mock.MagicMock()
+ scenario_cfg = {
+ 'runner': {
+ 'run_step': '',
+ },
+ }
+
+ runner = SearchRunner({})
+ runner._worker_run(cls, 'my_method', scenario_cfg, {})
+
+ @mock.patch('yardstick.benchmark.runners.search.multiprocessing')
+ def test__run_benchmark(self, mock_multi_process):
+ cls = mock.MagicMock()
+ scenario_cfg = {
+ 'runner': {},
+ }
+
+ runner = SearchRunner({})
+ runner._run_benchmark(cls, 'my_method', scenario_cfg, {})
+ self.assertEqual(mock_multi_process.Process.call_count, 1)
diff --git a/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py b/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
index e7ba3ca73..680692fdc 100644
--- a/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
+++ b/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
@@ -18,7 +18,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
@mock.patch('{}.GetNumaInfo._check_numa_node'.format(BASE))
@mock.patch('{}.GetNumaInfo._get_current_host_name'.format(BASE))
- @mock.patch('yaml.safe_load')
+ @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
@mock.patch('yardstick.common.task_template.TaskTemplate.render')
def test_get_numa_info(self,
mock_render,
@@ -44,7 +44,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
@mock.patch('yardstick.ssh.SSH.from_node')
@mock.patch('{}.GetNumaInfo._get_current_host_name'.format(BASE))
- @mock.patch('yaml.safe_load')
+ @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
@mock.patch('yardstick.common.task_template.TaskTemplate.render')
def test_check_numa_node(self,
mock_render,
@@ -74,7 +74,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
@mock.patch('{}.change_obj_to_dict'.format(BASE))
@mock.patch('{}.get_nova_client'.format(BASE))
- @mock.patch('yaml.safe_load')
+ @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
@mock.patch('yardstick.common.task_template.TaskTemplate.render')
def test_get_current_host_name(self,
mock_render,
diff --git a/tests/unit/benchmark/scenarios/networking/test_vsperf_dpdk.py b/tests/unit/benchmark/scenarios/networking/test_vsperf_dpdk.py
new file mode 100644
index 000000000..3b9f99b08
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/networking/test_vsperf_dpdk.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+
+# Copyright 2017 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Unittest for yardstick.benchmark.scenarios.networking.vsperf.VsperfDPDK
+
+from __future__ import absolute_import
+try:
+ from unittest import mock
+except ImportError:
+ import mock
+import unittest
+
+from yardstick.benchmark.scenarios.networking import vsperf_dpdk
+
+
+@mock.patch('yardstick.benchmark.scenarios.networking.vsperf_dpdk.subprocess')
+@mock.patch('yardstick.benchmark.scenarios.networking.vsperf_dpdk.ssh')
+@mock.patch("yardstick.benchmark.scenarios.networking.vsperf_dpdk.open",
+ mock.mock_open())
+class VsperfDPDKTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.ctx = {
+ "host": {
+ "ip": "10.229.47.137",
+ "user": "ubuntu",
+ "password": "ubuntu",
+ },
+ }
+ self.args = {
+ 'task_id': "1234-5678",
+ 'options': {
+ 'testname': 'pvp_tput',
+ 'traffic_type': 'rfc2544_throughput',
+ 'frame_size': '64',
+ 'test_params': 'TRAFFICGEN_DURATION=30;',
+ 'trafficgen_port1': 'ens4',
+ 'trafficgen_port2': 'ens5',
+ 'conf_file': 'vsperf-yardstick.conf',
+ 'setup_script': 'setup_yardstick.sh',
+ 'moongen_helper_file': '~/moongen.py',
+ 'moongen_host_ip': '10.5.201.151',
+ 'moongen_port1_mac': '8c:dc:d4:ae:7c:5c',
+ 'moongen_port2_mac': '8c:dc:d4:ae:7c:5d',
+ 'trafficgen_port1_nw': 'test2',
+ 'trafficgen_port2_nw': 'test3',
+ },
+ 'sla': {
+ 'metrics': 'throughput_rx_fps',
+ 'throughput_rx_fps': 500000,
+ 'action': 'monitor',
+ }
+ }
+
+ def test_vsperf_dpdk_setup(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ def test_vsperf_dpdk_teardown(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ p.teardown()
+ self.assertEqual(p.setup_done, False)
+
+ def test_vsperf_dpdk_is_dpdk_setup_no(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ # is_dpdk_setup() specific mocks
+ mock_ssh.SSH.from_node().execute.return_value = (0, 'dummy', '')
+
+ result = p._is_dpdk_setup()
+ self.assertEqual(result, False)
+
+ def test_vsperf_dpdk_is_dpdk_setup_yes(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ # is_dpdk_setup() specific mocks
+ mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+ result = p._is_dpdk_setup()
+ self.assertEqual(result, True)
+
+ def test_vsperf_dpdk_dpdk_setup_first(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ # is_dpdk_setup() specific mocks
+ mock_ssh.SSH.from_node().execute.return_value = (0, 'dummy', '')
+
+ p.dpdk_setup()
+ self.assertEqual(p._is_dpdk_setup(), False)
+ self.assertEqual(p.dpdk_setup_done, True)
+
+ def test_vsperf_dpdk_dpdk_setup_next(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ # dpdk_setup() specific mocks
+ mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+ p.dpdk_setup()
+ self.assertEqual(p._is_dpdk_setup(), True)
+ self.assertEqual(p.dpdk_setup_done, True)
+
+ def test_vsperf_dpdk_dpdk_setup_fail(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ # dpdk_setup() specific mocks
+ mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+ self.assertRaises(RuntimeError, p.dpdk_setup)
+
+ def test_vsperf_dpdk_run_ok(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ # run() specific mocks
+ mock_subprocess.call().execute.return_value = None
+ mock_subprocess.call().execute.return_value = None
+ mock_ssh.SSH.from_node().execute.return_value = (
+ 0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
+
+ result = {}
+ p.run(result)
+
+ self.assertEqual(result['throughput_rx_fps'], '14797660.000')
+
+ def test_vsperf_dpdk_run_falied_vsperf_execution(self, mock_ssh,
+ mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ # run() specific mocks
+ mock_subprocess.call().execute.return_value = None
+ mock_subprocess.call().execute.return_value = None
+ mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+ result = {}
+ self.assertRaises(RuntimeError, p.run, result)
+
+ def test_vsperf_dpdk_run_falied_csv_report(self, mock_ssh, mock_subprocess):
+ p = vsperf_dpdk.VsperfDPDK(self.args, self.ctx)
+
+ # setup() specific mocks
+ mock_subprocess.call().execute.return_value = None
+
+ p.setup()
+ self.assertIsNotNone(p.client)
+ self.assertEqual(p.setup_done, True)
+
+ # run() specific mocks
+ mock_subprocess.call().execute.return_value = None
+ mock_subprocess.call().execute.return_value = None
+ mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+ mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+ result = {}
+ self.assertRaises(RuntimeError, p.run, result)
+
+def main():
+ unittest.main()
+
+
+if __name__ == '__main__':
+ main()