diff options
5 files changed, 505 insertions, 0 deletions
diff --git a/yardstick/vTC/apexlake/experimental_framework/benchmarks/benchmark_base_class.py b/yardstick/vTC/apexlake/experimental_framework/benchmarks/benchmark_base_class.py new file mode 100644 index 000000000..756962714 --- /dev/null +++ b/yardstick/vTC/apexlake/experimental_framework/benchmarks/benchmark_base_class.py @@ -0,0 +1,80 @@ +# Copyright (c) 2015 Intel Research and Development Ireland Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import abc + + +class BenchmarkBaseClass(object): + ''' + This class represents a Benchmark that we want to run on the platform. + One of them will be the calculation of the throughput changing the + configuration parameters + ''' + + def __init__(self, name, params): + if not params: + params = dict() + if not isinstance(params, dict): + raise ValueError("Parameters need to be provided in a dict") + + for param in self.get_features()['parameters']: + if param not in params.keys(): + params[param] = self.get_features()['default_values'][param] + + for param in self.get_features()['parameters']: + if params[param] not in \ + (self.get_features())['allowed_values'][param]: + raise ValueError('Value of parameter "' + param + + '" is not allowed') + self.name = name + self.params = params + + def get_name(self): + return self.name + + def get_features(self): + features = dict() + features['description'] = 'Please implement the method ' \ + '"get_features" for your benchmark' + features['parameters'] = list() + features['allowed_values'] = dict() + features['default_values'] = dict() + return features + + @abc.abstractmethod + def init(self): + """ + Initializes the benchmark + :return: + """ + raise NotImplementedError("Subclass must implement abstract method") + + @abc.abstractmethod + def finalize(self): + """ + Finalizes the benchmark + :return: + """ + raise NotImplementedError("Subclass must implement abstract method") + + @abc.abstractmethod + def run(self): + """ + This method executes the specific benchmark on the VNF already + instantiated + :return: list of dictionaries (every dictionary contains the results + of a data point + """ + raise NotImplementedError("Subclass must implement abstract method") diff --git a/yardstick/vTC/apexlake/experimental_framework/benchmarks/rfc2544_throughput_benchmark.py b/yardstick/vTC/apexlake/experimental_framework/benchmarks/rfc2544_throughput_benchmark.py new file mode 100644 index 000000000..2ac3ea9c4 --- /dev/null +++ b/yardstick/vTC/apexlake/experimental_framework/benchmarks/rfc2544_throughput_benchmark.py @@ -0,0 +1,143 @@ +# Copyright (c) 2015 Intel Research and Development Ireland Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from experimental_framework.benchmarks import benchmark_base_class +from experimental_framework.packet_generators \ + import dpdk_packet_generator as dpdk +import experimental_framework.common as common +from experimental_framework.constants import framework_parameters as fp + + +PACKET_SIZE = 'packet_size' +VLAN_SENDER = 'vlan_sender' +VLAN_RECEIVER = 'vlan_receiver' + + +class RFC2544ThroughputBenchmark(benchmark_base_class.BenchmarkBaseClass): + """ + Calculates the throughput of the VNF under test according to the RFC2544. + """ + + def __init__(self, name, params): + benchmark_base_class.BenchmarkBaseClass.__init__(self, name, params) + self.base_dir = common.get_base_dir() + \ + fp.EXPERIMENTAL_FRAMEWORK_DIR + fp.DPDK_PKTGEN_DIR + self.results_file = self.base_dir + 'experiment.res' + self.lua_file = self.base_dir + 'rfc2544.lua' + + def init(self): + """ + Initialize the benchmark + :return: None + """ + pass + + def finalize(self): + """ + :return: None + """ + pass + + def get_features(self): + """ + Returns the features associated to the benchmark + :return: + """ + features = dict() + features['description'] = 'RFC 2544 Throughput calculation' + features['parameters'] = [PACKET_SIZE, VLAN_SENDER, VLAN_RECEIVER] + features['allowed_values'] = dict() + features['allowed_values'][PACKET_SIZE] = ['64', '128', '256', '512', + '1024', '1280', '1514'] + features['allowed_values'][VLAN_SENDER] = map(str, range(-1, 4096)) + features['allowed_values'][VLAN_RECEIVER] = map(str, range(-1, 4096)) + features['default_values'] = dict() + features['default_values'][PACKET_SIZE] = '1280' + features['default_values'][VLAN_SENDER] = '1007' + features['default_values'][VLAN_RECEIVER] = '1006' + return features + + def run(self): + """ + Sends and receive traffic according to the RFC methodology in order + to measure the throughput of the workload + :return: Results of the testcase (type: dict) + """ + ret_val = dict() + packet_size = self._extract_packet_size_from_params() + ret_val[PACKET_SIZE] = packet_size + + # Packetgen management + packetgen = dpdk.DpdkPacketGenerator() + self._configure_lua_file() + packetgen.init_dpdk_pktgen(dpdk_interfaces=2, + pcap_file_0='packet_' + + packet_size + '.pcap', + pcap_file_1='igmp.pcap', + lua_script='rfc2544.lua', + vlan_0=self.params[VLAN_SENDER], + vlan_1=self.params[VLAN_RECEIVER]) + common.LOG.debug('Start the packet generator - packet size: ' + + str(packet_size)) + packetgen.send_traffic() + common.LOG.debug('Stop the packet generator') + + # Result Collection + results = self._get_results() + for metric_name in results.keys(): + ret_val[metric_name] = results[metric_name] + self._reset_lua_file() + return ret_val + + def _extract_packet_size_from_params(self): + """ + Extracts packet sizes from parameters + :return: packet_sizes (list) + """ + packet_size = '1280' # default value + if PACKET_SIZE in self.params.keys() and \ + isinstance(self.params[PACKET_SIZE], str): + packet_size = self.params[PACKET_SIZE] + return packet_size + + def _configure_lua_file(self): + """ + Configure the packet gen to write the results into the right file + :return: None + """ + common.replace_in_file(self.lua_file, 'local out_file = ""', + 'local out_file = "' + + self.results_file + '"') + + def _reset_lua_file(self): + """ + Sets back the configuration of the local file var to the default + :return: + """ + common.replace_in_file(self.lua_file, 'local out_file = "' + + self.results_file + '"', + 'local out_file = ""') + + def _get_results(self): + """ + Returns the results of the experiment + :return: None + """ + throughput = common.get_file_first_line(self.results_file) + ret_val = dict() + try: + ret_val['throughput'] = int(throughput) + except: + ret_val['throughput'] = 0 + return ret_val diff --git a/yardstick/vTC/apexlake/experimental_framework/benchmarks/test_benchmark.py b/yardstick/vTC/apexlake/experimental_framework/benchmarks/test_benchmark.py new file mode 100644 index 000000000..d530168da --- /dev/null +++ b/yardstick/vTC/apexlake/experimental_framework/benchmarks/test_benchmark.py @@ -0,0 +1,36 @@ +# Copyright (c) 2015 Intel Research and Development Ireland Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from experimental_framework.benchmarks import benchmark_base_class as base + + +class TestBenchmark(base.BenchmarkBaseClass): + + def init(self): + pass + + def finalize(self): + pass + + def get_features(self): + features = dict() + features['description'] = 'Test Benchmark' + features['parameters'] = list() + features['allowed_values'] = dict() + features['default_values'] = dict() + return features + + def run(self): + return dict() diff --git a/yardstick/vTC/apexlake/tests/benchmark_base_class_test.py b/yardstick/vTC/apexlake/tests/benchmark_base_class_test.py new file mode 100644 index 000000000..405c0102f --- /dev/null +++ b/yardstick/vTC/apexlake/tests/benchmark_base_class_test.py @@ -0,0 +1,85 @@ +# Copyright (c) 2015 Intel Research and Development Ireland Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest +from experimental_framework.benchmarks import benchmark_base_class as base + + +class DummyBechmarkBaseClass(base.BenchmarkBaseClass): + + def get_features(self): + features = dict() + features['description'] = '***' + features['parameters'] = ['A', 'B'] + features['allowed_values'] = dict() + features['allowed_values']['A'] = ['a'] + features['allowed_values']['B'] = ['b'] + features['default_values'] = dict() + features['default_values']['A'] = 'a' + features['default_values']['B'] = 'b' + return features + + +class TestBenchmarkBaseClass(unittest.TestCase): + + def setUp(self): + self.mut = base.BenchmarkBaseClass('name', dict()) + + def test_constructor_for_success(self): + name = 'name' + params = dict() + params['A'] = 'a' + params['B'] = 'b' + params['C'] = 'c' + bench_base = DummyBechmarkBaseClass(name, params) + self.assertEqual(name, bench_base.name) + self.assertIn('A', bench_base.params.keys()) + self.assertIn('B', bench_base.params.keys()) + self.assertEqual('a', bench_base.params['A']) + self.assertEqual('b', bench_base.params['B']) + + params = dict() + params['A'] = 'a' + # params['B'] = 'b' + bench_base = DummyBechmarkBaseClass(name, params) + # self.assertEqual(name, bench_base.name) + # self.assertIn('A', bench_base.params.keys()) + # self.assertIn('B', bench_base.params.keys()) + # self.assertEqual('a', bench_base.params['A']) + # self.assertEqual('b', bench_base.params['B']) + + def test_constructor_for_failure(self): + name = 'name' + params = 'params' + self.assertRaises(ValueError, DummyBechmarkBaseClass, name, params) + + def test_constructor_for_failure_2(self): + name = 'name' + params = dict() + params['A'] = 'a' + params['B'] = '*' + self.assertRaises(ValueError, DummyBechmarkBaseClass, name, params) + + def test_init_for_failure(self): + self.assertRaises(NotImplementedError, self.mut.init) + + def test_finalize_for_failure(self): + self.assertRaises(NotImplementedError, self.mut.finalize) + + def test_run_for_failure(self): + self.assertRaises(NotImplementedError, self.mut.run) + + def test_get_name_for_success(self): + self.assertEqual(self.mut.get_name(), 'name') diff --git a/yardstick/vTC/apexlake/tests/rfc2544_throughput_benchmark_test.py b/yardstick/vTC/apexlake/tests/rfc2544_throughput_benchmark_test.py new file mode 100644 index 000000000..bef9b7f30 --- /dev/null +++ b/yardstick/vTC/apexlake/tests/rfc2544_throughput_benchmark_test.py @@ -0,0 +1,161 @@ +# Copyright (c) 2015 Intel Research and Development Ireland Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = 'vmriccox' + + +import unittest +import mock +from experimental_framework.benchmarks import rfc2544_throughput_benchmark \ + as mut +import experimental_framework.common as common + + +class RFC2544ThroughputBenchmarkRunTest(unittest.TestCase): + + def setUp(self): + name = 'benchmark' + params = dict() + params[mut.VLAN_SENDER] = '1' + params[mut.VLAN_RECEIVER] = '2' + self.benchmark = mut.RFC2544ThroughputBenchmark(name, params) + common.init_log() + + def tearDown(self): + pass + + def test_get_features_for_sanity(self): + output = self.benchmark.get_features() + self.assertIsInstance(output, dict) + self.assertIn('parameters', output.keys()) + self.assertIn('allowed_values', output.keys()) + self.assertIn('default_values', output.keys()) + self.assertIsInstance(output['parameters'], list) + self.assertIsInstance(output['allowed_values'], dict) + self.assertIsInstance(output['default_values'], dict) + + def test_init(self): + self.assertEqual(self.benchmark.init(), None) + + def test_finalize(self): + self.assertEqual(self.benchmark.finalize(), None) + + @mock.patch('experimental_framework.benchmarks.' + 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark.' + '_reset_lua_file') + @mock.patch('experimental_framework.benchmarks.' + 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark.' + '_configure_lua_file') + @mock.patch('experimental_framework.benchmarks.' + 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark.' + '_extract_packet_size_from_params') + @mock.patch('experimental_framework.benchmarks.' + 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark.' + '_get_results') + @mock.patch('experimental_framework.benchmarks.' + 'rfc2544_throughput_benchmark.dpdk.DpdkPacketGenerator') + def test_run_for_success(self, mock_dpdk, mock_get_results, + mock_extract_size, conf_lua_file_mock, + reset_lua_file_mock): + expected = {'results': 0, 'packet_size': '1'} + mock_extract_size.return_value = '1' + mock_get_results.return_value = {'results': 0} + output = self.benchmark.run() + self.assertEqual(expected, output) + conf_lua_file_mock.assert_called_once() + reset_lua_file_mock.assert_called_once() + dpdk_instance = mock_dpdk() + dpdk_instance.init_dpdk_pktgen.assert_called_once_with( + dpdk_interfaces=2, pcap_file_0='packet_1.pcap', + pcap_file_1='igmp.pcap', lua_script='rfc2544.lua', + vlan_0='1', vlan_1='2') + dpdk_instance.send_traffic.assert_called_once_with() + + +class RFC2544ThroughputBenchmarkOthers(unittest.TestCase): + + def setUp(self): + name = 'benchmark' + params = {'packet_size': '128'} + self.benchmark = mut.RFC2544ThroughputBenchmark(name, params) + + def tearDown(self): + pass + + def test__extract_packet_size_from_params_for_success(self): + expected = '128' + output = self.benchmark._extract_packet_size_from_params() + self.assertEqual(expected, output) + + @mock.patch('experimental_framework.common.replace_in_file') + def test__configure_lua_file(self, mock_common_replace_in_file): + self.benchmark.lua_file = 'lua_file' + self.benchmark.results_file = 'result_file' + self.benchmark._configure_lua_file() + mock_common_replace_in_file.\ + assert_called_once_with('lua_file', 'local out_file = ""', + 'local out_file = "result_file"') + + @mock.patch('experimental_framework.common.replace_in_file') + def test__reset_lua_file(self, mock_common_replace_in_file): + self.benchmark.lua_file = 'lua_file' + self.benchmark.results_file = 'result_file' + self.benchmark._reset_lua_file() + mock_common_replace_in_file.\ + assert_called_once_with('lua_file', + 'local out_file = "result_file"', + 'local out_file = ""') + + +class RFC2544ThroughputBenchmarkGetResultsTest(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + @mock.patch('experimental_framework.common.get_file_first_line') + def test__get_results_for_success(self, mock_common_file_line): + name = 'benchmark' + params = {'packet_size': '128'} + self.benchmark = mut.RFC2544ThroughputBenchmark(name, params) + self.benchmark.results_file = 'base_dir/experimental_framework/' \ + 'packet_generators/dpdk_pktgen/' \ + 'experiment.res' + mock_common_file_line.return_value = '10' + expected = {'throughput': 10} + output = self.benchmark._get_results() + self.assertEqual(expected, output) + mock_common_file_line.\ + assert_called_once_with('base_dir/experimental_framework/' + 'packet_generators/dpdk_pktgen/' + 'experiment.res') + + @mock.patch('experimental_framework.common.get_file_first_line') + def test__get_results_for_success_2(self, mock_common_file_line): + name = 'benchmark' + params = {'packet_size': '128'} + self.benchmark = mut.RFC2544ThroughputBenchmark(name, params) + self.benchmark.results_file = 'base_dir/experimental_framework/' \ + 'packet_generators/dpdk_pktgen/' \ + 'experiment.res' + mock_common_file_line.return_value = '1XXX0' + expected = {'throughput': 0} + output = self.benchmark._get_results() + self.assertEqual(expected, output) + mock_common_file_line.\ + assert_called_once_with('base_dir/experimental_framework/' + 'packet_generators/dpdk_pktgen/' + 'experiment.res') |