aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--yardstick/vTC/apexlake/experimental_framework/api.py24
-rw-r--r--yardstick/vTC/apexlake/experimental_framework/benchmarking_unit.py2
-rw-r--r--yardstick/vTC/apexlake/tests/api_test.py57
-rw-r--r--yardstick/vTC/apexlake/tests/benchmarking_unit_test.py157
-rw-r--r--yardstick/vTC/apexlake/tests/experiment_test.py94
-rw-r--r--yardstick/vTC/apexlake/tests/instantiation_validation_noisy_bench_test.py4
6 files changed, 125 insertions, 213 deletions
diff --git a/yardstick/vTC/apexlake/experimental_framework/api.py b/yardstick/vTC/apexlake/experimental_framework/api.py
index 635dcd2cf..1851f1b09 100644
--- a/yardstick/vTC/apexlake/experimental_framework/api.py
+++ b/yardstick/vTC/apexlake/experimental_framework/api.py
@@ -27,18 +27,18 @@ class FrameworkApi(object):
"""
common.init(api=True)
- @staticmethod
- def get_available_test_cases():
- """
- Returns a list of available test cases.
- This list include eventual modules developed by the user, if any.
- Each test case is returned as a string that represents the full name
- of the test case and that can be used to get more information
- calling get_test_case_features(test_case_name)
-
- :return: list of strings
- """
- return b_unit.BenchmarkingUnit.get_available_test_cases()
+ # @staticmethod
+ # def get_available_test_cases():
+ # """
+ # Returns a list of available test cases.
+ # This list include eventual modules developed by the user, if any.
+ # Each test case is returned as a string that represents the full name
+ # of the test case and that can be used to get more information
+ # calling get_test_case_features(test_case_name)
+ #
+ # :return: list of strings
+ # """
+ # return b_unit.BenchmarkingUnit.get_available_test_cases()
@staticmethod
def get_test_case_features(test_case):
diff --git a/yardstick/vTC/apexlake/experimental_framework/benchmarking_unit.py b/yardstick/vTC/apexlake/experimental_framework/benchmarking_unit.py
index 1a19826ca..1963696f8 100644
--- a/yardstick/vTC/apexlake/experimental_framework/benchmarking_unit.py
+++ b/yardstick/vTC/apexlake/experimental_framework/benchmarking_unit.py
@@ -188,7 +188,7 @@ class BenchmarkingUnit:
for key in benchmark.get_params():
experiment[key] = benchmark.get_params()[key]
common.LOG.info('Benchmark Finished')
- self.data_manager.generate_result_csv_file()
+ # self.data_manager.generate_result_csv_file()
common.LOG.info('Benchmarking Unit: Experiments completed!')
return result
diff --git a/yardstick/vTC/apexlake/tests/api_test.py b/yardstick/vTC/apexlake/tests/api_test.py
index e3d5a8b2c..4b70b9bd6 100644
--- a/yardstick/vTC/apexlake/tests/api_test.py
+++ b/yardstick/vTC/apexlake/tests/api_test.py
@@ -80,35 +80,34 @@ class TestGeneratesTemplate(unittest.TestCase):
# output = FrameworkApi.get_available_test_cases()
# self.assertEqual(expected, output)
- # @mock.patch('experimental_framework.benchmarking_unit.BenchmarkingUnit.'
- # 'get_required_benchmarks',
- # side_effect=DummyBenchmarkingUnit.get_required_benchmarks)
- # def test_get_test_case_features_for_success(self, mock_get_req_bench):
- #
- # expected = dict()
- # expected['description'] = 'Instantiation Validation Benchmark'
- # expected['parameters'] = [
- # iv.THROUGHPUT,
- # iv.VLAN_SENDER,
- # iv.VLAN_RECEIVER]
- # expected['allowed_values'] = dict()
- # expected['allowed_values'][iv.THROUGHPUT] = \
- # map(str, range(0, 100))
- # expected['allowed_values'][iv.VLAN_SENDER] = \
- # map(str, range(-1, 4096))
- # expected['allowed_values'][iv.VLAN_RECEIVER] = \
- # map(str, range(-1, 4096))
- # expected['default_values'] = dict()
- # expected['default_values'][iv.THROUGHPUT] = '1'
- # expected['default_values'][iv.VLAN_SENDER] = '-1'
- # expected['default_values'][iv.VLAN_RECEIVER] = '-1'
- #
- # test_case = 'instantiation_validation_benchmark.' \
- # 'InstantiationValidationBenchmark'
- # output = FrameworkApi.get_test_case_features(test_case)
- # self.assertEqual(expected, output)
-
- def test____for_failure(self):
+ @mock.patch('experimental_framework.benchmarking_unit.BenchmarkingUnit.'
+ 'get_required_benchmarks',
+ side_effect=DummyBenchmarkingUnit.get_required_benchmarks)
+ def test_get_test_case_features_for_success(self, mock_get_req_bench):
+ expected = dict()
+ expected['description'] = 'Instantiation Validation Benchmark'
+ expected['parameters'] = [
+ iv.THROUGHPUT,
+ iv.VLAN_SENDER,
+ iv.VLAN_RECEIVER]
+ expected['allowed_values'] = dict()
+ expected['allowed_values'][iv.THROUGHPUT] = \
+ map(str, range(0, 100))
+ expected['allowed_values'][iv.VLAN_SENDER] = \
+ map(str, range(-1, 4096))
+ expected['allowed_values'][iv.VLAN_RECEIVER] = \
+ map(str, range(-1, 4096))
+ expected['default_values'] = dict()
+ expected['default_values'][iv.THROUGHPUT] = '1'
+ expected['default_values'][iv.VLAN_SENDER] = '-1'
+ expected['default_values'][iv.VLAN_RECEIVER] = '-1'
+
+ test_case = 'instantiation_validation_benchmark.' \
+ 'InstantiationValidationBenchmark'
+ output = FrameworkApi.get_test_case_features(test_case)
+ self.assertEqual(expected, output)
+
+ def test__get_test_case_features__for_failure(self):
self.assertRaises(
ValueError, FrameworkApi.get_test_case_features, 111)
diff --git a/yardstick/vTC/apexlake/tests/benchmarking_unit_test.py b/yardstick/vTC/apexlake/tests/benchmarking_unit_test.py
index b0f800a4d..ccf64066a 100644
--- a/yardstick/vTC/apexlake/tests/benchmarking_unit_test.py
+++ b/yardstick/vTC/apexlake/tests/benchmarking_unit_test.py
@@ -18,59 +18,59 @@ __author__ = 'vmriccox'
import unittest
import mock
from experimental_framework.benchmarking_unit import BenchmarkingUnit
-from experimental_framework.data_manager import DataManager
+# from experimental_framework.data_manager import DataManager
from experimental_framework.deployment_unit import DeploymentUnit
import experimental_framework.common as common
from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
RFC2544ThroughputBenchmark
-class DummyDataManager(DataManager):
-
- def __init__(self, experiment_directory):
- self.experiment_directory = experiment_directory
- self.experiments = dict()
- self.new_exp_counter = 0
- self.add_bench_counter = 0
- self.close_experiment_1_counter = 0
- self.close_experiment_2_counter = 0
- self.generate_csv_counter = 0
-
- def create_new_experiment(self, experiment_name, get_counter=None):
- if not get_counter:
- self.new_exp_counter += 1
- else:
- return self.new_exp_counter
-
- def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
- if not get_counter:
- self.add_bench_counter += 1
- else:
- return self.add_bench_counter
-
- def close_experiment(self, experiment, get_counter=None):
- if get_counter:
- return [self.close_experiment_1_counter,
- self.close_experiment_2_counter]
- if experiment == 'VTC_base_single_vm_wait_1':
- self.close_experiment_1_counter += 1
- if experiment == 'VTC_base_single_vm_wait_2':
- self.close_experiment_2_counter += 1
-
- def generate_result_csv_file(self, get_counter=None):
- if get_counter:
- return self.generate_csv_counter
- else:
- self.generate_csv_counter += 1
-
- def add_metadata(self, experiment_name, metadata):
- pass
-
- def add_configuration(self, experiment_name, configuration):
- pass
-
- def add_data_points(self, experiment_name, benchmark_name, result):
- pass
+# class DummyDataManager(DataManager):
+#
+# def __init__(self, experiment_directory):
+# self.experiment_directory = experiment_directory
+# self.experiments = dict()
+# self.new_exp_counter = 0
+# self.add_bench_counter = 0
+# self.close_experiment_1_counter = 0
+# self.close_experiment_2_counter = 0
+# self.generate_csv_counter = 0
+#
+# def create_new_experiment(self, experiment_name, get_counter=None):
+# if not get_counter:
+# self.new_exp_counter += 1
+# else:
+# return self.new_exp_counter
+#
+# def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
+# if not get_counter:
+# self.add_bench_counter += 1
+# else:
+# return self.add_bench_counter
+#
+# def close_experiment(self, experiment, get_counter=None):
+# if get_counter:
+# return [self.close_experiment_1_counter,
+# self.close_experiment_2_counter]
+# if experiment == 'VTC_base_single_vm_wait_1':
+# self.close_experiment_1_counter += 1
+# if experiment == 'VTC_base_single_vm_wait_2':
+# self.close_experiment_2_counter += 1
+#
+# def generate_result_csv_file(self, get_counter=None):
+# if get_counter:
+# return self.generate_csv_counter
+# else:
+# self.generate_csv_counter += 1
+#
+# def add_metadata(self, experiment_name, metadata):
+# pass
+#
+# def add_configuration(self, experiment_name, configuration):
+# pass
+#
+# def add_data_points(self, experiment_name, benchmark_name, result):
+# pass
class Dummy_2544(RFC2544ThroughputBenchmark):
@@ -122,12 +122,13 @@ class TestBenchmarkingUnit(unittest.TestCase):
@mock.patch('time.time')
@mock.patch('experimental_framework.common.get_template_dir')
- @mock.patch('experimental_framework.data_manager.DataManager',
- side_effect=DummyDataManager)
+ # @mock.patch('experimental_framework.data_manager.DataManager',
+ # side_effect=DummyDataManager)
@mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
@mock.patch('experimental_framework.benchmarking_unit.heat.'
'get_all_heat_templates')
- def test___init__(self, mock_heat, mock_dep_unit, mock_data_manager,
+ def test___init__(self, mock_heat, mock_dep_unit,
+ # mock_data_manager,
mock_temp_dir, mock_time):
mock_heat.return_value = list()
mock_time.return_value = '12345'
@@ -152,7 +153,7 @@ class TestBenchmarkingUnit(unittest.TestCase):
benchmarks)
self.assertEqual(bu.required_benchmarks, benchmarks)
bu.heat_template_parameters = heat_template_parameters
- mock_data_manager.assert_called_once_with('tests/data/results/12345')
+ # mock_data_manager.assert_called_once_with('tests/data/results/12345')
mock_dep_unit.assert_called_once_with(openstack_credentials)
mock_heat.assert_called_once_with('tests/data/results/', '.ext')
@@ -160,13 +161,14 @@ class TestBenchmarkingUnit(unittest.TestCase):
'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
@mock.patch('time.time')
@mock.patch('experimental_framework.common.get_template_dir')
- @mock.patch('experimental_framework.data_manager.DataManager',
- side_effect=DummyDataManager)
+ # @mock.patch('experimental_framework.data_manager.DataManager',
+ # side_effect=DummyDataManager)
@mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
@mock.patch('experimental_framework.benchmarking_unit.'
'heat.get_all_heat_templates')
def test_initialize_for_success(self, mock_heat, mock_dep_unit,
- mock_data_manager, mock_temp_dir,
+ # mock_data_manager,
+ mock_temp_dir,
mock_time, mock_rfc2544):
mock_heat.return_value = list()
mock_time.return_value = '12345'
@@ -204,21 +206,22 @@ class TestBenchmarkingUnit(unittest.TestCase):
self.assertTrue(len(bu.benchmarks) == 1)
self.assertEqual(bu.benchmarks[0].__class__,
Dummy_2544)
- self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
- self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
+ # self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
+ # self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
@mock.patch('experimental_framework.benchmarks.'
'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
@mock.patch('time.time')
@mock.patch('experimental_framework.common.get_template_dir')
- @mock.patch('experimental_framework.data_manager.DataManager',
- side_effect=DummyDataManager)
+ # @mock.patch('experimental_framework.data_manager.DataManager',
+ # side_effect=DummyDataManager)
@mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
@mock.patch('experimental_framework.benchmarking_unit.'
'heat.get_all_heat_templates')
def test_finalize_for_success(
- self, mock_heat, mock_dep_unit, mock_data_manager, mock_temp_dir,
- mock_time, mock_rfc2544):
+ self, mock_heat, mock_dep_unit,
+ # mock_data_manager,
+ mock_temp_dir, mock_time, mock_rfc2544):
mock_heat.return_value = list()
mock_time.return_value = '12345'
mock_temp_dir.return_value = 'tests/data/test_templates/'
@@ -252,7 +255,7 @@ class TestBenchmarkingUnit(unittest.TestCase):
'VTC_base_single_vm_wait_2.yaml']
bu.finalize()
# self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
- self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
+ # self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
@mock.patch('experimental_framework.common.push_data_influxdb')
@mock.patch('experimental_framework.common.LOG')
@@ -260,14 +263,15 @@ class TestBenchmarkingUnit(unittest.TestCase):
'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
@mock.patch('time.time')
@mock.patch('experimental_framework.common.get_template_dir')
- @mock.patch('experimental_framework.data_manager.DataManager',
- side_effect=DummyDataManager)
+ # @mock.patch('experimental_framework.data_manager.DataManager',
+ # side_effect=DummyDataManager)
@mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
@mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
@mock.patch('experimental_framework.benchmarking_unit.'
'heat.get_all_heat_templates')
def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
- mock_dep_unit, mock_data_manager,
+ mock_dep_unit,
+ # mock_data_manager,
mock_temp_dir, mock_time,
mock_rfc2544, mock_log, mock_influx):
mock_heat.return_value = list()
@@ -301,7 +305,7 @@ class TestBenchmarkingUnit(unittest.TestCase):
heat_template_parameters,
iterations,
benchmarks)
- bu.data_manager = DummyDataManager('tests/data/results/12345')
+ # bu.data_manager = DummyDataManager('tests/data/results/12345')
bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
'VTC_base_single_vm_wait_2.yaml']
bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
@@ -320,15 +324,16 @@ class TestBenchmarkingUnit(unittest.TestCase):
'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
@mock.patch('time.time')
@mock.patch('experimental_framework.common.get_template_dir')
- @mock.patch('experimental_framework.data_manager.DataManager',
- side_effect=DummyDataManager)
+ # @mock.patch('experimental_framework.data_manager.DataManager',
+ # side_effect=DummyDataManager)
@mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
@mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
@mock.patch('experimental_framework.benchmarking_unit.'
'heat.get_all_heat_templates')
def test_run_benchmarks_2_for_success(
self, mock_heat, mock_common_dep_unit, mock_dep_unit,
- mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
+ # mock_data_manager,
+ mock_temp_dir, mock_time, mock_rfc2544,
mock_log):
mock_heat.return_value = list()
mock_time.return_value = '12345'
@@ -358,7 +363,7 @@ class TestBenchmarkingUnit(unittest.TestCase):
heat_template_parameters,
iterations,
benchmarks)
- bu.data_manager = DummyDataManager('tests/data/results/12345')
+ # bu.data_manager = DummyDataManager('tests/data/results/12345')
bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
'VTC_base_single_vm_wait_2.yaml']
bu.benchmarks = [Dummy_2544('dummy', dict())]
@@ -373,15 +378,16 @@ class TestBenchmarkingUnit(unittest.TestCase):
'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
@mock.patch('time.time')
@mock.patch('experimental_framework.common.get_template_dir')
- @mock.patch('experimental_framework.data_manager.DataManager',
- side_effect=DummyDataManager)
+ # @mock.patch('experimental_framework.data_manager.DataManager',
+ # side_effect=DummyDataManager)
@mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
@mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
@mock.patch('experimental_framework.benchmarking_unit.'
'heat.get_all_heat_templates')
def test_get_benchmark_name_for_success(
self, mock_heat, mock_common_dep_unit, mock_dep_unit,
- mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
+ # mock_data_manager,
+ mock_temp_dir, mock_time, mock_rfc2544,
mock_log):
mock_heat.return_value = list()
mock_time.return_value = '12345'
@@ -427,15 +433,16 @@ class TestBenchmarkingUnit(unittest.TestCase):
'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
@mock.patch('time.time')
@mock.patch('experimental_framework.common.get_template_dir')
- @mock.patch('experimental_framework.data_manager.DataManager',
- side_effect=DummyDataManager)
+ # @mock.patch('experimental_framework.data_manager.DataManager',
+ # side_effect=DummyDataManager)
@mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
@mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
@mock.patch('experimental_framework.benchmarking_unit.'
'heat.get_all_heat_templates')
def test_get_required_benchmarks_for_success(
self, mock_heat, mock_common_dep_unit, mock_dep_unit,
- mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
+ # mock_data_manager,
+ mock_temp_dir, mock_time, mock_rfc2544,
mock_log):
mock_heat.return_value = list()
mock_time.return_value = '12345'
diff --git a/yardstick/vTC/apexlake/tests/experiment_test.py b/yardstick/vTC/apexlake/tests/experiment_test.py
deleted file mode 100644
index 47d1fbb77..000000000
--- a/yardstick/vTC/apexlake/tests/experiment_test.py
+++ /dev/null
@@ -1,94 +0,0 @@
-# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-__author__ = 'gpetralx'
-
-import unittest
-from experimental_framework import data_manager
-
-
-class TestExperiment(unittest.TestCase):
- def setUp(self):
- self.exp = data_manager.Experiment('experiment_1')
-
- def tearDown(self):
- pass
-
- def test_add_experiment_metadata(self):
- with self.assertRaises(ValueError):
- self.exp.add_experiment_metadata('metadata')
-
- metadata = {
- 'item_1': 'value_1',
- 'item_2': 'value_2',
- 'item_3': 'value_3'
- }
- self.exp.add_experiment_metadata(metadata)
- self.assertDictEqual(metadata, self.exp._metadata)
- self.assertDictEqual(metadata, self.exp.get_metadata())
-
- def test_experiment_configuration(self):
- with self.assertRaises(ValueError):
- self.exp.add_experiment_configuration('configuration')
- configuration = {
- 'item_1': 'value_1',
- 'item_2': 'value_2',
- 'item_3': 'value_3'
- }
- self.exp.add_experiment_configuration(configuration)
- self.assertDictEqual(configuration, self.exp._configuration)
- self.assertDictEqual(configuration, self.exp.get_configuration())
-
- def test_add_benchmark(self):
- with self.assertRaises(ValueError):
- self.exp.add_benchmark(1)
- self.exp.add_benchmark('benchmark_1')
- self.assertListEqual(list(), self.exp._benchmarks['benchmark_1'])
-
- def test_add_datapoint(self):
- with self.assertRaises(ValueError):
- self.exp.add_data_point('benchmark_1', 'datapoint')
-
- data_point_1 = {
- 'point_1': 'value_1',
- 'point_2': 'value_2',
- 'point_3': 'value_3'
- }
-
- with self.assertRaises(ValueError):
- self.exp.add_data_point('benchmark_1', data_point_1)
-
- self.exp.add_benchmark('benchmark_1')
- self.exp.add_data_point('benchmark_1', data_point_1)
- self.assertListEqual([data_point_1],
- self.exp._benchmarks['benchmark_1'])
-
- def test_get_data_points(self):
- self.assertListEqual(list(), self.exp.get_data_points('benchmark_1'))
- data_point_1 = {
- 'point_1': 'value_1',
- 'point_2': 'value_2',
- 'point_3': 'value_3'
- }
- self.exp.add_benchmark('benchmark_1')
- self.exp.add_data_point('benchmark_1', data_point_1)
- self.assertListEqual([data_point_1],
- self.exp.get_data_points('benchmark_1'))
-
- def test_get_benchmarks(self):
- self.exp.add_benchmark('benchmark_1')
- self.exp.add_benchmark('benchmark_2')
- self.exp.add_benchmark('benchmark_3')
- expected = ['benchmark_3', 'benchmark_2', 'benchmark_1']
- self.assertListEqual(expected, self.exp.get_benchmarks())
diff --git a/yardstick/vTC/apexlake/tests/instantiation_validation_noisy_bench_test.py b/yardstick/vTC/apexlake/tests/instantiation_validation_noisy_bench_test.py
index cdcce37e3..463035743 100644
--- a/yardstick/vTC/apexlake/tests/instantiation_validation_noisy_bench_test.py
+++ b/yardstick/vTC/apexlake/tests/instantiation_validation_noisy_bench_test.py
@@ -78,14 +78,14 @@ class InstantiationValidationInitTest(unittest.TestCase):
expected['allowed_values'][mut.NUMBER_OF_CORES] = \
['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10']
expected['allowed_values'][mut.AMOUNT_OF_RAM] = \
- ['250M', '1G', '2G', '3G', '4G', '5G', '6G', '7G', '8G', '9G',
+ ['256M', '1G', '2G', '3G', '4G', '5G', '6G', '7G', '8G', '9G',
'10G']
expected['default_values']['throughput'] = '1'
expected['default_values']['vlan_sender'] = '-1'
expected['default_values']['vlan_receiver'] = '-1'
expected['default_values'][mut.NUM_OF_NEIGHBORS] = '1'
expected['default_values'][mut.NUMBER_OF_CORES] = '1'
- expected['default_values'][mut.AMOUNT_OF_RAM] = '250M'
+ expected['default_values'][mut.AMOUNT_OF_RAM] = '256M'
output = self.iv.get_features()
self.assertEqual(expected['description'], output['description'])