diff options
| -rw-r--r-- | storperf/utilities/data_treatment.py | 39 | ||||
| -rw-r--r-- | storperf/utilities/math.py | 27 | ||||
| -rw-r--r-- | storperf/utilities/steady_state.py | 45 | ||||
| -rw-r--r-- | tests/utilities_tests/data_treatment_test.py | 81 | ||||
| -rw-r--r-- | tests/utilities_tests/math_average_test.py | 52 | ||||
| -rw-r--r-- | tests/utilities_tests/math_range_test.py | 48 | ||||
| -rw-r--r-- | tests/utilities_tests/math_slope_test.py | 24 | ||||
| -rw-r--r-- | tests/utilities_tests/steady_state_test.py | 59 |
8 files changed, 337 insertions, 38 deletions
diff --git a/storperf/utilities/data_treatment.py b/storperf/utilities/data_treatment.py new file mode 100644 index 0000000..2368fd9 --- /dev/null +++ b/storperf/utilities/data_treatment.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2016 CENGN and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +def data_treatment(data_series): + """ + This function aims at performing any necessary pre treatment on the + data_series passed to the steady_state function before being passed + under to the different math utilities (slope, range and average) + so the data can match the requirements of each algorithm. + The function returns a dictionary composed of three values that can be + accessed with the following keys : 'slope_data', 'range_data' and + 'average_data'. + The data_series is currently assumed to follow the pattern : + [[x1,y1], [x2,y2], ..., [xm,ym]]. If this pattern were to change, or + the input data pattern of one of the math module, this data_treatment + function should be the only part of the Steady State detection module + that would need to be modified too. + """ + + x_values = [] + y_values = [] + for l in data_series: + x_values.append(l[0]) + y_values.append(l[1]) + + treated_data = { + 'slope_data': data_series, # No treatment necessary so far + 'range_data': y_values, # The y_values only + 'average_data': y_values + } + + return treated_data diff --git a/storperf/utilities/math.py b/storperf/utilities/math.py index 031fc3e..a11ec19 100644 --- a/storperf/utilities/math.py +++ b/storperf/utilities/math.py @@ -25,7 +25,7 @@ def slope(data_series): # In the particular case of an empty data series if len(data_series) == 0: - beta2 = 0 + beta2 = None else: # The general case m = len(data_series) @@ -78,7 +78,7 @@ def range_value(data_series): # In the particular case of an empty data series if len(data_series) == 0: - range_value = 0 + range_value = None else: # The general case max_value = max(data_series) @@ -86,3 +86,26 @@ def range_value(data_series): range_value = max_value - min_value return range_value + + +def average(data_series): + """ + This function seeks to calculate the average value of the data series + given a series following the pattern : [y1, y2, y3, ..., ym]. + If this data pattern were to change, the data_treatment function + should be adjusted to ensure compatibility with the rest of the + Steady State Dectection module. + The function returns a float number corresponding to the average of the yi. + """ + m = len(data_series) + + if m == 0: # In the particular case of an empty data series + average = None + + else: + data_sum = 0 + for value in data_series: + data_sum += value + average = data_sum / float(m) + + return average diff --git a/storperf/utilities/steady_state.py b/storperf/utilities/steady_state.py new file mode 100644 index 0000000..8bfcb93 --- /dev/null +++ b/storperf/utilities/steady_state.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2016 CENGN and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from storperf.utilities import data_treatment as DataTreatment +from storperf.utilities import math as math + + +def steady_state(data_series): + """ + This function seeks to detect steady state given on a measurement + window given the data series of that measurement window following + the pattern : [[x1,y1], [x2,y2], ..., [xm,ym]]. m represents the number + of points recorded in the measurement window, x which represents the + time, and y which represents the Volume performance variable being + tested e.g. IOPS, latency... + The function returns a boolean describing wether or not steady state + has been reached with the data that is passed to it. + """ + + # Pre conditioning the data to match the algorithms + treated_data = DataTreatment.data_treatment(data_series) + + # Calculating useful values invoking dedicated functions + slope_value = math.slope(treated_data['slope_data']) + range_value = math.range_value(treated_data['range_data']) + average_value = math.average(treated_data['average_data']) + + if (slope_value is not None and range_value is not None and + average_value is not None): + # Verification of the Steady State conditions following the SNIA + # definition + range_condition = range_value < 0.20 * abs(average_value) + slope_condition = slope_value < 0.10 * abs(average_value) + + steady_state = range_condition and slope_condition + + else: + steady_state = False + + return steady_state diff --git a/tests/utilities_tests/data_treatment_test.py b/tests/utilities_tests/data_treatment_test.py new file mode 100644 index 0000000..4450f92 --- /dev/null +++ b/tests/utilities_tests/data_treatment_test.py @@ -0,0 +1,81 @@ +############################################################################## +# Copyright (c) 2016 CENGN and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import unittest +from storperf.utilities import data_treatment as DataTreatment + + +class DataTreatmentTest(unittest.TestCase): + + def setUp(self): + unittest.TestCase.setUp(self) + + def test_empty_series(self): + expected = { + 'slope_data': [], + 'range_data': [], + 'average_data': [] + } + data_series = [] + actual = DataTreatment.data_treatment(data_series) + self.assertEqual(expected, actual) + + def test_integer_series(self): + expected = { + 'slope_data': [[1, 5], [66, 2], [12, 98], [74, 669], [33, 66]], + 'range_data': [5, 2, 98, 669, 66], + 'average_data': [5, 2, 98, 669, 66] + } + data_series = [[1, 5], [66, 2], [12, 98], [74, 669], [33, 66]] + actual = DataTreatment.data_treatment(data_series) + self.assertEqual(expected, actual) + + def test_float_series(self): + expected = { + 'slope_data': [[5.6, 12.7], [96.66, 78.212], + [639.568, 5.3], [4.65, 6.667]], + 'range_data': [12.7, 78.212, 5.3, 6.667], + 'average_data': [12.7, 78.212, 5.3, 6.667] + } + data_series = [ + [5.6, 12.7], [96.66, 78.212], [639.568, 5.3], [4.65, 6.667]] + actual = DataTreatment.data_treatment(data_series) + self.assertEqual(expected, actual) + + def test_float_int_mix(self): + expected = { + 'slope_data': [[5, 12.7], [96.66, 7], [639.568, 5.3], [4, 6]], + 'range_data': [12.7, 7, 5.3, 6], + 'average_data': [12.7, 7, 5.3, 6] + } + data_series = [[5, 12.7], [96.66, 7], [639.568, 5.3], [4, 6]] + actual = DataTreatment.data_treatment(data_series) + self.assertEqual(expected, actual) + + def test_negative_values(self): + expected = { + 'slope_data': [[-15, 5.56], [41.3, -278], [41.3, -98], + [78.336, -0.12], [33.667, 66]], + 'range_data': [5.56, -278, -98, -0.12, 66], + 'average_data': [5.56, -278, -98, -0.12, 66] + } + data_series = [ + [-15, 5.56], [41.3, -278], [41.3, -98], + [78.336, -0.12], [33.667, 66]] + actual = DataTreatment.data_treatment(data_series) + self.assertEqual(expected, actual) + + def test_single_value(self): + expected = { + 'slope_data': [[86.8, 65.36]], + 'range_data': [65.36], + 'average_data': [65.36] + } + data_series = [[86.8, 65.36]] + actual = DataTreatment.data_treatment(data_series) + self.assertEqual(expected, actual) diff --git a/tests/utilities_tests/math_average_test.py b/tests/utilities_tests/math_average_test.py new file mode 100644 index 0000000..3095f56 --- /dev/null +++ b/tests/utilities_tests/math_average_test.py @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2016 CENGN and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import unittest +from storperf.utilities import math as math + + +class MathAverageTest(unittest.TestCase): + + def setUp(self): + unittest.TestCase.setUp(self) + + def test_empty_series(self): + expected = None + data_series = [] + actual = math.average(data_series) + self.assertEqual(expected, actual) + + def test_integer_series(self): + expected = 19.75 + data_series = [5, 12, 7, 55] + actual = math.average(data_series) + self.assertEqual(expected, actual) + + def test_float_series(self): + expected = 63.475899999999996 + data_series = [78.6, 45.187, 33.334, 96.7826] + actual = math.average(data_series) + self.assertEqual(expected, actual) + + def test_float_int_mix(self): + expected = 472.104 + data_series = [10, 557.33, 862, 56.99, 874.2] + actual = math.average(data_series) + self.assertEqual(expected, actual) + + def test_negative_values(self): + expected = -17.314 + data_series = [-15.654, 59.5, 16.25, -150, 3.334] + actual = math.average(data_series) + self.assertEqual(expected, actual) + + def test_single_value(self): + expected = -66.6667 + data_series = [-66.6667] + actual = math.average(data_series) + self.assertEqual(expected, actual) diff --git a/tests/utilities_tests/math_range_test.py b/tests/utilities_tests/math_range_test.py index 6484752..90519e7 100644 --- a/tests/utilities_tests/math_range_test.py +++ b/tests/utilities_tests/math_range_test.py @@ -9,7 +9,7 @@ from random import uniform, randrange import unittest -from storperf.utilities import math as math +from storperf.utilities import math as Range class MathRangeTest(unittest.TestCase): @@ -18,56 +18,56 @@ class MathRangeTest(unittest.TestCase): unittest.TestCase.setUp(self) def test_empty_series(self): - expected = 0 + expected = None data_series = [] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_integer_series(self): expected = 11946 data_series = [5, 351, 847, 2, 1985, 18, 96, 389, 687, 1, 11947, 758, 155] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_float_series_1_decimal(self): expected = 778595.5 data_series = [736.4, 9856.4, 684.2, 0.3, 0.9, 778595.8] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_float_series_2_decimals(self): expected = 5693.47 data_series = [51.36, 78.40, 1158.24, 5.50, 0.98, 5694.45] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_float_series_3_decimals(self): expected = 992.181 data_series = [4.562, 12.582, 689.452, 135.162, 996.743, 65.549, 36.785] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_float_series_4_decimals(self): expected = 122985.3241 data_series = [39.4785, 896.7845, 11956.3654, 44.2398, 6589.7134, 0.3671, 122985.6912] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_float_series_5_decimals(self): expected = 8956208.84494 data_series = [12.78496, 55.91275, 668.94378, 550396.5671, 512374.9999, 8956221.6299] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_float_series_10_decimals(self): expected = 5984.507397972699 data_series = [1.1253914785, 5985.6327894512, 256.1875693287, 995.8497623415] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_float_mix(self): @@ -75,46 +75,46 @@ class MathRangeTest(unittest.TestCase): data_series = [60785.9962, 899.4, 78.66, 69.58, 4.93795, 587.195486, 96.7694536, 5.13755964, 33.333333334, 60786.5624872199] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_float_integer_mix(self): expected = 460781.05825 data_series = [460785.9962, 845.634, 24.1, 69.58, 89, 4.93795] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_negative_values(self): expected = 596.78163 data_series = [-4.655, -33.3334, -596.78422, -0.00259, -66.785] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_negative_positive_mix(self): expected = 58.859500000000004 data_series = [6.85698, -2.8945, 0, -0.15, 55.965] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_single_element(self): expected = 0 data_series = [2.265] - actual = math.range_value(data_series) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_10000_values_processing(self): expected = 28001.068 - data_series = [uniform(-10000, 10000) for i in xrange(10000)] - data_series.insert(randrange(len(data_series) + 1), 15000.569) - data_series.insert(randrange(len(data_series) + 1), -13000.499) - actual = math.range_value(data_series) + data_series = [uniform(-10000, 10000) for _ in range(10000)] + data_series.insert(randrange(len(data_series)), 15000.569) + data_series.insert(randrange(len(data_series)), -13000.499) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) def test_processing_100_values_100_times(self): expected = 35911.3134 - for index in range(1, 100): - data_series = [uniform(-10000, 10000) for i in xrange(100)] - data_series.insert(randrange(len(data_series) + 1), 16956.3334) - data_series.insert(randrange(len(data_series) + 1), -18954.98) - actual = math.range_value(data_series) + for _ in range(1, 100): + data_series = [uniform(-10000, 10000) for _ in range(100)] + data_series.insert(randrange(len(data_series)), 16956.3334) + data_series.insert(randrange(len(data_series)), -18954.98) + actual = Range.range_value(data_series) self.assertEqual(expected, actual) diff --git a/tests/utilities_tests/math_slope_test.py b/tests/utilities_tests/math_slope_test.py index a34845b..6c05aa4 100644 --- a/tests/utilities_tests/math_slope_test.py +++ b/tests/utilities_tests/math_slope_test.py @@ -7,7 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import unittest -from storperf.utilities import math as math +from storperf.utilities import math as Slope class MathSlopeTest(unittest.TestCase): @@ -17,51 +17,51 @@ class MathSlopeTest(unittest.TestCase): pass def test_slope_empty_series(self): - expected = 0 - actual = math.slope([]) + expected = None + actual = Slope.slope([]) self.assertEqual(expected, actual) def test_slope_integer_series(self): expected = 1.4 - actual = math.slope([[1, 6], [2, 5], [3, 7], [4, 10]]) + actual = Slope.slope([[1, 6], [2, 5], [3, 7], [4, 10]]) self.assertEqual(expected, actual) def test_slope_decimal_series(self): expected = 1.4 - actual = math.slope([[1.0, 6.0], [2.0, 5.0], [3.0, 7.0], [4.0, 10.0]]) + actual = Slope.slope([[1.0, 6.0], [2.0, 5.0], [3.0, 7.0], [4.0, 10.0]]) self.assertEqual(expected, actual) def test_slope_decimal_integer_mix(self): expected = 1.4 - actual = math.slope([[1.0, 6], [2, 5.0], [3, 7], [4.0, 10]]) + actual = Slope.slope([[1.0, 6], [2, 5.0], [3, 7], [4.0, 10]]) self.assertEqual(expected, actual) def test_slope_negative_y_series(self): expected = 2 - actual = math.slope([[1.0, -2], [2, 2], [3, 2]]) + actual = Slope.slope([[1.0, -2], [2, 2], [3, 2]]) self.assertEqual(expected, actual) def test_slope_negative_x_series(self): expected = 1.4 - actual = math.slope([[-24, 6.0], [-23, 5], [-22, 7.0], [-21, 10]]) + actual = Slope.slope([[-24, 6.0], [-23, 5], [-22, 7.0], [-21, 10]]) self.assertEqual(expected, actual) def test_slope_out_of_order_series(self): expected = 1.4 - actual = math.slope([[2, 5.0], [4, 10], [3.0, 7], [1, 6]]) + actual = Slope.slope([[2, 5.0], [4, 10], [3.0, 7], [1, 6]]) self.assertEqual(expected, actual) def test_slope_0_in_y(self): expected = -0.5 - actual = math.slope([[15.5, 1], [16.5, 0], [17.5, 0]]) + actual = Slope.slope([[15.5, 1], [16.5, 0], [17.5, 0]]) self.assertEqual(expected, actual) def test_slope_0_in_x(self): expected = 1.4 - actual = math.slope([[0, 6.0], [1, 5], [2, 7], [3, 10]]) + actual = Slope.slope([[0, 6.0], [1, 5], [2, 7], [3, 10]]) self.assertEqual(expected, actual) def test_slope_0_in_x_and_y(self): expected = 1.5 - actual = math.slope([[0.0, 0], [1, 1], [2, 3]]) + actual = Slope.slope([[0.0, 0], [1, 1], [2, 3]]) self.assertEqual(expected, actual) diff --git a/tests/utilities_tests/steady_state_test.py b/tests/utilities_tests/steady_state_test.py new file mode 100644 index 0000000..d80c60d --- /dev/null +++ b/tests/utilities_tests/steady_state_test.py @@ -0,0 +1,59 @@ +############################################################################## +# Copyright (c) 2016 CENGN and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import unittest +from storperf.utilities import steady_state as SteadyState + + +class SteadyStateTest(unittest.TestCase): + + def setUp(self): + unittest.TestCase.setUp(self) + + def test_integer_values(self): + expected = True + data_series = [[305, 20], [306, 21], [307, 21], [308, 19]] + actual = SteadyState.steady_state(data_series) + self.assertEqual(expected, actual) + + def test_float_values(self): + expected = True + data_series = [ + [55.5, 40.5], [150.2, 42.3], [150.8, 41.8], [151.2, 41.5]] + actual = SteadyState.steady_state(data_series) + self.assertEqual(expected, actual) + + def test_float_integer_mix_false(self): + expected = False + data_series = [[1, 2], [2, 2.2], [3, 1.8], [4, 1.8]] + actual = SteadyState.steady_state(data_series) + self.assertEqual(expected, actual) + + def test_float_integer_mix_true(self): + expected = True + data_series = [[12, 18], [12.5, 18.2], [13, 16.8], [15, 16.8]] + actual = SteadyState.steady_state(data_series) + self.assertEqual(expected, actual) + + def test_empty_series(self): + expected = False + data_series = [] + actual = SteadyState.steady_state(data_series) + self.assertEqual(expected, actual) + + def test_negative_values(self): + expected = True + data_series = [[-1, -24.2], [0.5, -23.8], [1.1, -24.0], [3.2, -24.0]] + actual = SteadyState.steady_state(data_series) + self.assertEqual(expected, actual) + + def test_out_of_order_series(self): + expected = True + data_series = [[-15, 0.43], [-16, 0.41], [-3, 0.45], [4, 0.42]] + actual = SteadyState.steady_state(data_series) + self.assertEqual(expected, actual) |