Add support to the test case required by YARDSTICK-35

Add to ApexLake the support to the calculation of the throughput for the vTC using DPDK pktgen.

JIRA: YARDSTICK-35

Change-Id: I2ead9522648d6955f64fd18c543dabf7f26e2490
Signed-off-by: Vincenzo Riccobene <vincenzo.m.riccobene@intel.com>
Signed-off-by: Vincenzo Riccobene <vincenzox.m.riccobene@intel.com>

5 files changed, 505 insertions, 0 deletions

diff --git a/yardstick/vTC/apexlake/experimental_framework/benchmarks/benchmark_base_class.py b/yardstick/vTC/apexlake/experimental_framework/benchmarks/benchmark_base_class.py
new file mode 100644
index 000000000..756962714
--- /dev/null
+++ b/yardstick/vTC/apexlake/experimental_framework/benchmarks/benchmark_base_class.py
@@ -0,0 +1,80 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import abc
+
+
+class BenchmarkBaseClass(object):
+    '''
+    This class represents a Benchmark that we want to run on the platform.
+    One of them will be the calculation of the throughput changing the
+    configuration parameters
+    '''
+
+    def __init__(self, name, params):
+        if not params:
+            params = dict()
+        if not isinstance(params, dict):
+            raise ValueError("Parameters need to be provided in a dict")
+
+        for param in self.get_features()['parameters']:
+            if param not in params.keys():
+                params[param] = self.get_features()['default_values'][param]
+
+        for param in self.get_features()['parameters']:
+            if params[param] not in \
+                    (self.get_features())['allowed_values'][param]:
+                raise ValueError('Value of parameter "' + param +
+                                 '" is not allowed')
+        self.name = name
+        self.params = params
+
+    def get_name(self):
+        return self.name
+
+    def get_features(self):
+        features = dict()
+        features['description'] = 'Please implement the method ' \
+                                  '"get_features" for your benchmark'
+        features['parameters'] = list()
+        features['allowed_values'] = dict()
+        features['default_values'] = dict()
+        return features
+
+    @abc.abstractmethod
+    def init(self):
+        """
+        Initializes the benchmark
+        :return:
+        """
+        raise NotImplementedError("Subclass must implement abstract method")
+
+    @abc.abstractmethod
+    def finalize(self):
+        """
+        Finalizes the benchmark
+        :return:
+        """
+        raise NotImplementedError("Subclass must implement abstract method")
+
+    @abc.abstractmethod
+    def run(self):
+        """
+        This method executes the specific benchmark on the VNF already
+        instantiated
+        :return: list of dictionaries (every dictionary contains the results
+        of a data point
+        """
+        raise NotImplementedError("Subclass must implement abstract method")
diff --git a/yardstick/vTC/apexlake/experimental_framework/benchmarks/rfc2544_throughput_benchmark.py b/yardstick/vTC/apexlake/experimental_framework/benchmarks/rfc2544_throughput_benchmark.py
new file mode 100644
index 000000000..2ac3ea9c4
--- /dev/null
+++ b/yardstick/vTC/apexlake/experimental_framework/benchmarks/rfc2544_throughput_benchmark.py
@@ -0,0 +1,143 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from experimental_framework.benchmarks import benchmark_base_class
+from experimental_framework.packet_generators \
+    import dpdk_packet_generator as dpdk
+import experimental_framework.common as common
+from experimental_framework.constants import framework_parameters as fp
+
+
+PACKET_SIZE = 'packet_size'
+VLAN_SENDER = 'vlan_sender'
+VLAN_RECEIVER = 'vlan_receiver'
+
+
+class RFC2544ThroughputBenchmark(benchmark_base_class.BenchmarkBaseClass):
+    """
+    Calculates the throughput of the VNF under test according to the RFC2544.
+    """
+
+    def __init__(self, name, params):
+        benchmark_base_class.BenchmarkBaseClass.__init__(self, name, params)
+        self.base_dir = common.get_base_dir() + \
+            fp.EXPERIMENTAL_FRAMEWORK_DIR + fp.DPDK_PKTGEN_DIR
+        self.results_file = self.base_dir + 'experiment.res'
+        self.lua_file = self.base_dir + 'rfc2544.lua'
+
+    def init(self):
+        """
+        Initialize the benchmark
+        :return: None
+        """
+        pass
+
+    def finalize(self):
+        """
+        :return: None
+        """
+        pass
+
+    def get_features(self):
+        """
+        Returns the features associated to the benchmark
+        :return:
+        """
+        features = dict()
+        features['description'] = 'RFC 2544 Throughput calculation'
+        features['parameters'] = [PACKET_SIZE, VLAN_SENDER, VLAN_RECEIVER]
+        features['allowed_values'] = dict()
+        features['allowed_values'][PACKET_SIZE] = ['64', '128', '256', '512',
+                                                   '1024', '1280', '1514']
+        features['allowed_values'][VLAN_SENDER] = map(str, range(-1, 4096))
+        features['allowed_values'][VLAN_RECEIVER] = map(str, range(-1, 4096))
+        features['default_values'] = dict()
+        features['default_values'][PACKET_SIZE] = '1280'
+        features['default_values'][VLAN_SENDER] = '1007'
+        features['default_values'][VLAN_RECEIVER] = '1006'
+        return features
+
+    def run(self):
+        """
+        Sends and receive traffic according to the RFC methodology in order
+        to measure the throughput of the workload
+        :return: Results of the testcase (type: dict)
+        """
+        ret_val = dict()
+        packet_size = self._extract_packet_size_from_params()
+        ret_val[PACKET_SIZE] = packet_size
+
+        # Packetgen management
+        packetgen = dpdk.DpdkPacketGenerator()
+        self._configure_lua_file()
+        packetgen.init_dpdk_pktgen(dpdk_interfaces=2,
+                                   pcap_file_0='packet_' +
+                                               packet_size + '.pcap',
+                                   pcap_file_1='igmp.pcap',
+                                   lua_script='rfc2544.lua',
+                                   vlan_0=self.params[VLAN_SENDER],
+                                   vlan_1=self.params[VLAN_RECEIVER])
+        common.LOG.debug('Start the packet generator - packet size: ' +
+                         str(packet_size))
+        packetgen.send_traffic()
+        common.LOG.debug('Stop the packet generator')
+
+        # Result Collection
+        results = self._get_results()
+        for metric_name in results.keys():
+            ret_val[metric_name] = results[metric_name]
+        self._reset_lua_file()
+        return ret_val
+
+    def _extract_packet_size_from_params(self):
+        """
+        Extracts packet sizes from parameters
+        :return: packet_sizes (list)
+        """
+        packet_size = '1280'  # default value
+        if PACKET_SIZE in self.params.keys() and \
+                isinstance(self.params[PACKET_SIZE], str):
+            packet_size = self.params[PACKET_SIZE]
+        return packet_size
+
+    def _configure_lua_file(self):
+        """
+        Configure the packet gen to write the results into the right file
+        :return: None
+        """
+        common.replace_in_file(self.lua_file, 'local out_file = ""',
+                               'local out_file = "' +
+                               self.results_file + '"')
+
+    def _reset_lua_file(self):
+        """
+        Sets back the configuration of the local file var to the default
+        :return:
+        """
+        common.replace_in_file(self.lua_file, 'local out_file = "' +
+                               self.results_file + '"',
+                               'local out_file = ""')
+
+    def _get_results(self):
+        """
+        Returns the results of the experiment
+        :return: None
+        """
+        throughput = common.get_file_first_line(self.results_file)
+        ret_val = dict()
+        try:
+            ret_val['throughput'] = int(throughput)
+        except:
+            ret_val['throughput'] = 0
+        return ret_val
diff --git a/yardstick/vTC/apexlake/experimental_framework/benchmarks/test_benchmark.py b/yardstick/vTC/apexlake/experimental_framework/benchmarks/test_benchmark.py
new file mode 100644
index 000000000..d530168da
--- /dev/null
+++ b/yardstick/vTC/apexlake/experimental_framework/benchmarks/test_benchmark.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from experimental_framework.benchmarks import benchmark_base_class as base
+
+
+class TestBenchmark(base.BenchmarkBaseClass):
+
+    def init(self):
+        pass
+
+    def finalize(self):
+        pass
+
+    def get_features(self):
+        features = dict()
+        features['description'] = 'Test Benchmark'
+        features['parameters'] = list()
+        features['allowed_values'] = dict()
+        features['default_values'] = dict()
+        return features
+
+    def run(self):
+        return dict()
diff --git a/yardstick/vTC/apexlake/tests/benchmark_base_class_test.py b/yardstick/vTC/apexlake/tests/benchmark_base_class_test.py
new file mode 100644
index 000000000..405c0102f
--- /dev/null
+++ b/yardstick/vTC/apexlake/tests/benchmark_base_class_test.py
@@ -0,0 +1,85 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+from experimental_framework.benchmarks import benchmark_base_class as base
+
+
+class DummyBechmarkBaseClass(base.BenchmarkBaseClass):
+
+    def get_features(self):
+        features = dict()
+        features['description'] = '***'
+        features['parameters'] = ['A', 'B']
+        features['allowed_values'] = dict()
+        features['allowed_values']['A'] = ['a']
+        features['allowed_values']['B'] = ['b']
+        features['default_values'] = dict()
+        features['default_values']['A'] = 'a'
+        features['default_values']['B'] = 'b'
+        return features
+
+
+class TestBenchmarkBaseClass(unittest.TestCase):
+
+    def setUp(self):
+        self.mut = base.BenchmarkBaseClass('name', dict())
+
+    def test_constructor_for_success(self):
+        name = 'name'
+        params = dict()
+        params['A'] = 'a'
+        params['B'] = 'b'
+        params['C'] = 'c'
+        bench_base = DummyBechmarkBaseClass(name, params)
+        self.assertEqual(name, bench_base.name)
+        self.assertIn('A', bench_base.params.keys())
+        self.assertIn('B', bench_base.params.keys())
+        self.assertEqual('a', bench_base.params['A'])
+        self.assertEqual('b', bench_base.params['B'])
+
+        params = dict()
+        params['A'] = 'a'
+        # params['B'] = 'b'
+        bench_base = DummyBechmarkBaseClass(name, params)
+        # self.assertEqual(name, bench_base.name)
+        # self.assertIn('A', bench_base.params.keys())
+        # self.assertIn('B', bench_base.params.keys())
+        # self.assertEqual('a', bench_base.params['A'])
+        # self.assertEqual('b', bench_base.params['B'])
+
+    def test_constructor_for_failure(self):
+        name = 'name'
+        params = 'params'
+        self.assertRaises(ValueError, DummyBechmarkBaseClass, name, params)
+
+    def test_constructor_for_failure_2(self):
+        name = 'name'
+        params = dict()
+        params['A'] = 'a'
+        params['B'] = '*'
+        self.assertRaises(ValueError, DummyBechmarkBaseClass, name, params)
+
+    def test_init_for_failure(self):
+        self.assertRaises(NotImplementedError, self.mut.init)
+
+    def test_finalize_for_failure(self):
+        self.assertRaises(NotImplementedError, self.mut.finalize)
+
+    def test_run_for_failure(self):
+        self.assertRaises(NotImplementedError, self.mut.run)
+
+    def test_get_name_for_success(self):
+        self.assertEqual(self.mut.get_name(), 'name')
diff --git a/yardstick/vTC/apexlake/tests/rfc2544_throughput_benchmark_test.py b/yardstick/vTC/apexlake/tests/rfc2544_throughput_benchmark_test.py
new file mode 100644
index 000000000..bef9b7f30
--- /dev/null
+++ b/yardstick/vTC/apexlake/tests/rfc2544_throughput_benchmark_test.py
@@ -0,0 +1,161 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'vmriccox'
+
+
+import unittest
+import mock
+from experimental_framework.benchmarks import rfc2544_throughput_benchmark \
+    as mut
+import experimental_framework.common as common
+
+
+class RFC2544ThroughputBenchmarkRunTest(unittest.TestCase):
+
+    def setUp(self):
+        name = 'benchmark'
+        params = dict()
+        params[mut.VLAN_SENDER] = '1'
+        params[mut.VLAN_RECEIVER] = '2'
+        self.benchmark = mut.RFC2544ThroughputBenchmark(name, params)
+        common.init_log()
+
+    def tearDown(self):
+        pass
+
+    def test_get_features_for_sanity(self):
+        output = self.benchmark.get_features()
+        self.assertIsInstance(output, dict)
+        self.assertIn('parameters', output.keys())
+        self.assertIn('allowed_values', output.keys())
+        self.assertIn('default_values', output.keys())
+        self.assertIsInstance(output['parameters'], list)
+        self.assertIsInstance(output['allowed_values'], dict)
+        self.assertIsInstance(output['default_values'], dict)
+
+    def test_init(self):
+        self.assertEqual(self.benchmark.init(), None)
+
+    def test_finalize(self):
+        self.assertEqual(self.benchmark.finalize(), None)
+
+    @mock.patch('experimental_framework.benchmarks.'
+                'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark.'
+                '_reset_lua_file')
+    @mock.patch('experimental_framework.benchmarks.'
+                'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark.'
+                '_configure_lua_file')
+    @mock.patch('experimental_framework.benchmarks.'
+                'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark.'
+                '_extract_packet_size_from_params')
+    @mock.patch('experimental_framework.benchmarks.'
+                'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark.'
+                '_get_results')
+    @mock.patch('experimental_framework.benchmarks.'
+                'rfc2544_throughput_benchmark.dpdk.DpdkPacketGenerator')
+    def test_run_for_success(self, mock_dpdk, mock_get_results,
+                             mock_extract_size, conf_lua_file_mock,
+                             reset_lua_file_mock):
+        expected = {'results': 0, 'packet_size': '1'}
+        mock_extract_size.return_value = '1'
+        mock_get_results.return_value = {'results': 0}
+        output = self.benchmark.run()
+        self.assertEqual(expected, output)
+        conf_lua_file_mock.assert_called_once()
+        reset_lua_file_mock.assert_called_once()
+        dpdk_instance = mock_dpdk()
+        dpdk_instance.init_dpdk_pktgen.assert_called_once_with(
+            dpdk_interfaces=2, pcap_file_0='packet_1.pcap',
+            pcap_file_1='igmp.pcap', lua_script='rfc2544.lua',
+            vlan_0='1', vlan_1='2')
+        dpdk_instance.send_traffic.assert_called_once_with()
+
+
+class RFC2544ThroughputBenchmarkOthers(unittest.TestCase):
+
+    def setUp(self):
+        name = 'benchmark'
+        params = {'packet_size': '128'}
+        self.benchmark = mut.RFC2544ThroughputBenchmark(name, params)
+
+    def tearDown(self):
+        pass
+
+    def test__extract_packet_size_from_params_for_success(self):
+        expected = '128'
+        output = self.benchmark._extract_packet_size_from_params()
+        self.assertEqual(expected, output)
+
+    @mock.patch('experimental_framework.common.replace_in_file')
+    def test__configure_lua_file(self, mock_common_replace_in_file):
+        self.benchmark.lua_file = 'lua_file'
+        self.benchmark.results_file = 'result_file'
+        self.benchmark._configure_lua_file()
+        mock_common_replace_in_file.\
+            assert_called_once_with('lua_file', 'local out_file = ""',
+                                    'local out_file = "result_file"')
+
+    @mock.patch('experimental_framework.common.replace_in_file')
+    def test__reset_lua_file(self, mock_common_replace_in_file):
+        self.benchmark.lua_file = 'lua_file'
+        self.benchmark.results_file = 'result_file'
+        self.benchmark._reset_lua_file()
+        mock_common_replace_in_file.\
+            assert_called_once_with('lua_file',
+                                    'local out_file = "result_file"',
+                                    'local out_file = ""')
+
+
+class RFC2544ThroughputBenchmarkGetResultsTest(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    @mock.patch('experimental_framework.common.get_file_first_line')
+    def test__get_results_for_success(self, mock_common_file_line):
+        name = 'benchmark'
+        params = {'packet_size': '128'}
+        self.benchmark = mut.RFC2544ThroughputBenchmark(name, params)
+        self.benchmark.results_file = 'base_dir/experimental_framework/' \
+                                      'packet_generators/dpdk_pktgen/' \
+                                      'experiment.res'
+        mock_common_file_line.return_value = '10'
+        expected = {'throughput': 10}
+        output = self.benchmark._get_results()
+        self.assertEqual(expected, output)
+        mock_common_file_line.\
+            assert_called_once_with('base_dir/experimental_framework/'
+                                    'packet_generators/dpdk_pktgen/'
+                                    'experiment.res')
+
+    @mock.patch('experimental_framework.common.get_file_first_line')
+    def test__get_results_for_success_2(self, mock_common_file_line):
+        name = 'benchmark'
+        params = {'packet_size': '128'}
+        self.benchmark = mut.RFC2544ThroughputBenchmark(name, params)
+        self.benchmark.results_file = 'base_dir/experimental_framework/' \
+                                      'packet_generators/dpdk_pktgen/' \
+                                      'experiment.res'
+        mock_common_file_line.return_value = '1XXX0'
+        expected = {'throughput': 0}
+        output = self.benchmark._get_results()
+        self.assertEqual(expected, output)
+        mock_common_file_line.\
+            assert_called_once_with('base_dir/experimental_framework/'
+                                    'packet_generators/dpdk_pktgen/'
+                                    'experiment.res')