From e708f56598414929091524ce68a56da602d082e7 Mon Sep 17 00:00:00 2001 From: mbeierl Date: Thu, 15 Oct 2015 10:48:58 -0400 Subject: Logging and timestamp Added logging and fixed issue if fio does not produce the current timestamp in its output JIRA: STORPERF-4 Change-Id: Ifd0dbc4e17d984907e63089ebfae1d0e9e749dcc Signed-off-by: mbeierl --- tests/carbon/json_to_carbon_test.py | 845 +++++++++++++++++++++++++++++++++++- 1 file changed, 837 insertions(+), 8 deletions(-) (limited to 'tests') diff --git a/tests/carbon/json_to_carbon_test.py b/tests/carbon/json_to_carbon_test.py index ea261af..060d93d 100644 --- a/tests/carbon/json_to_carbon_test.py +++ b/tests/carbon/json_to_carbon_test.py @@ -14,14 +14,14 @@ import json import carbon.converter class JSONToCarbonTest(unittest.TestCase): - + single_json_text_element = """{ "timestamp" : "timestamp", "key" : "value" }""" single_json_numeric_element = """{ "timestamp" : "timestamp", "key" : 123 }""" single_json_key_with_spaces = """{ "timestamp" : "timestamp", "key with spaces" : "value" }""" single_json_value_with_spaces = """{ "timestamp" : "timestamp", "key" : "value with spaces" }""" json_map_name_with_spaces = """{ "timestamp" : "timestamp", "map with spaces" : { "key" : "value" } }""" json_list_name_with_spaces = """{ "timestamp" : "timestamp", "list with spaces" : [{ "key" : "value" }] }""" - + sample_json = """ { "colorsArray":[{ @@ -68,6 +68,835 @@ class JSONToCarbonTest(unittest.TestCase): } }] } +""" + + full_fio = """ +{ + "fio version" : "fio-2.1.11", + "jobs" : [ + { + "jobname" : "random-read", + "groupid" : 0, + "error" : 0, + "read" : { + "io_bytes" : 44920, + "bw" : 21, + "iops" : 5, + "runtime" : 2127550, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 53, + "max" : 6329, + "mean" : 255.34, + "stddev" : 94.40, + "percentile" : { + "1.000000" : 84, + "5.000000" : 181, + "10.000000" : 203, + "20.000000" : 241, + "30.000000" : 241, + "40.000000" : 243, + "50.000000" : 243, + "60.000000" : 243, + "70.000000" : 251, + "80.000000" : 270, + "90.000000" : 318, + "95.000000" : 370, + "99.000000" : 506, + "99.500000" : 596, + "99.900000" : 812, + "99.950000" : 1032, + "99.990000" : 2992, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 53, + "max" : 6330, + "mean" : 255.56, + "stddev" : 94.41 + }, + "bw_min" : 15008, + "bw_max" : 15664, + "bw_agg" : 100.00, + "bw_mean" : 15433.60, + "bw_dev" : 264.57 + }, + "write" : { + "io_bytes" : 0, + "bw" : 0, + "iops" : 0, + "runtime" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00, + "percentile" : { + "1.000000" : 0, + "5.000000" : 0, + "10.000000" : 0, + "20.000000" : 0, + "30.000000" : 0, + "40.000000" : 0, + "50.000000" : 0, + "60.000000" : 0, + "70.000000" : 0, + "80.000000" : 0, + "90.000000" : 0, + "95.000000" : 0, + "99.000000" : 0, + "99.500000" : 0, + "99.900000" : 0, + "99.950000" : 0, + "99.990000" : 0, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.00, + "bw_mean" : 0.00, + "bw_dev" : 0.00 + }, + "trim" : { + "io_bytes" : 0, + "bw" : 0, + "iops" : 0, + "runtime" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00, + "percentile" : { + "1.000000" : 0, + "5.000000" : 0, + "10.000000" : 0, + "20.000000" : 0, + "30.000000" : 0, + "40.000000" : 0, + "50.000000" : 0, + "60.000000" : 0, + "70.000000" : 0, + "80.000000" : 0, + "90.000000" : 0, + "95.000000" : 0, + "99.000000" : 0, + "99.500000" : 0, + "99.900000" : 0, + "99.950000" : 0, + "99.990000" : 0, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.00, + "bw_mean" : 0.00, + "bw_dev" : 0.00 + }, + "usr_cpu" : 0.96, + "sys_cpu" : 23.82, + "ctx" : 11122, + "majf" : 0, + "minf" : 6, + "iodepth_level" : { + "1" : 100.00, + "2" : 0.00, + "4" : 0.00, + "8" : 0.00, + "16" : 0.00, + "32" : 0.00, + ">=64" : 0.00 + }, + "latency_us" : { + "2" : 0.00, + "4" : 0.00, + "10" : 0.00, + "20" : 0.00, + "50" : 0.00, + "100" : 1.31, + "250" : 67.24, + "500" : 30.34, + "750" : 0.94, + "1000" : 0.12 + }, + "latency_ms" : { + "2" : 0.03, + "4" : 0.02, + "10" : 0.01, + "20" : 0.00, + "50" : 0.00, + "100" : 0.00, + "250" : 0.00, + "500" : 0.00, + "750" : 0.00, + "1000" : 0.00, + "2000" : 0.00, + ">=2000" : 0.00 + }, + "latency_depth" : 2, + "latency_target" : 0, + "latency_percentile" : 100.00, + "latency_window" : 0 + }, + { + "jobname" : "random-read", + "groupid" : 0, + "error" : 0, + "read" : { + "io_bytes" : 44992, + "bw" : 21, + "iops" : 5, + "runtime" : 2119748, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 53, + "max" : 2976, + "mean" : 254.65, + "stddev" : 81.49, + "percentile" : { + "1.000000" : 79, + "5.000000" : 181, + "10.000000" : 195, + "20.000000" : 241, + "30.000000" : 241, + "40.000000" : 243, + "50.000000" : 243, + "60.000000" : 243, + "70.000000" : 251, + "80.000000" : 270, + "90.000000" : 318, + "95.000000" : 374, + "99.000000" : 510, + "99.500000" : 564, + "99.900000" : 972, + "99.950000" : 1496, + "99.990000" : 2800, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 53, + "max" : 2977, + "mean" : 254.87, + "stddev" : 81.50 + }, + "bw_min" : 15144, + "bw_max" : 15704, + "bw_agg" : 100.00, + "bw_mean" : 15467.20, + "bw_dev" : 222.90 + }, + "write" : { + "io_bytes" : 0, + "bw" : 0, + "iops" : 0, + "runtime" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00, + "percentile" : { + "1.000000" : 0, + "5.000000" : 0, + "10.000000" : 0, + "20.000000" : 0, + "30.000000" : 0, + "40.000000" : 0, + "50.000000" : 0, + "60.000000" : 0, + "70.000000" : 0, + "80.000000" : 0, + "90.000000" : 0, + "95.000000" : 0, + "99.000000" : 0, + "99.500000" : 0, + "99.900000" : 0, + "99.950000" : 0, + "99.990000" : 0, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.00, + "bw_mean" : 0.00, + "bw_dev" : 0.00 + }, + "trim" : { + "io_bytes" : 0, + "bw" : 0, + "iops" : 0, + "runtime" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00, + "percentile" : { + "1.000000" : 0, + "5.000000" : 0, + "10.000000" : 0, + "20.000000" : 0, + "30.000000" : 0, + "40.000000" : 0, + "50.000000" : 0, + "60.000000" : 0, + "70.000000" : 0, + "80.000000" : 0, + "90.000000" : 0, + "95.000000" : 0, + "99.000000" : 0, + "99.500000" : 0, + "99.900000" : 0, + "99.950000" : 0, + "99.990000" : 0, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.00, + "bw_mean" : 0.00, + "bw_dev" : 0.00 + }, + "usr_cpu" : 2.07, + "sys_cpu" : 22.74, + "ctx" : 11126, + "majf" : 0, + "minf" : 7, + "iodepth_level" : { + "1" : 100.00, + "2" : 0.00, + "4" : 0.00, + "8" : 0.00, + "16" : 0.00, + "32" : 0.00, + ">=64" : 0.00 + }, + "latency_us" : { + "2" : 0.00, + "4" : 0.00, + "10" : 0.00, + "20" : 0.00, + "50" : 0.00, + "100" : 1.53, + "250" : 67.31, + "500" : 30.06, + "750" : 0.90, + "1000" : 0.12 + }, + "latency_ms" : { + "2" : 0.07, + "4" : 0.02, + "10" : 0.00, + "20" : 0.00, + "50" : 0.00, + "100" : 0.00, + "250" : 0.00, + "500" : 0.00, + "750" : 0.00, + "1000" : 0.00, + "2000" : 0.00, + ">=2000" : 0.00 + }, + "latency_depth" : 2, + "latency_target" : 0, + "latency_percentile" : 100.00, + "latency_window" : 0 + }, + { + "jobname" : "random-read", + "groupid" : 0, + "error" : 0, + "read" : { + "io_bytes" : 45024, + "bw" : 21, + "iops" : 5, + "runtime" : 2114297, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 53, + "max" : 1752, + "mean" : 254.25, + "stddev" : 72.36, + "percentile" : { + "1.000000" : 76, + "5.000000" : 181, + "10.000000" : 197, + "20.000000" : 241, + "30.000000" : 241, + "40.000000" : 243, + "50.000000" : 243, + "60.000000" : 245, + "70.000000" : 251, + "80.000000" : 270, + "90.000000" : 318, + "95.000000" : 374, + "99.000000" : 510, + "99.500000" : 564, + "99.900000" : 844, + "99.950000" : 1176, + "99.990000" : 1720, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 53, + "max" : 1752, + "mean" : 254.48, + "stddev" : 72.40 + }, + "bw_min" : 15320, + "bw_max" : 15784, + "bw_agg" : 100.00, + "bw_mean" : 15504.00, + "bw_dev" : 191.33 + }, + "write" : { + "io_bytes" : 0, + "bw" : 0, + "iops" : 0, + "runtime" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00, + "percentile" : { + "1.000000" : 0, + "5.000000" : 0, + "10.000000" : 0, + "20.000000" : 0, + "30.000000" : 0, + "40.000000" : 0, + "50.000000" : 0, + "60.000000" : 0, + "70.000000" : 0, + "80.000000" : 0, + "90.000000" : 0, + "95.000000" : 0, + "99.000000" : 0, + "99.500000" : 0, + "99.900000" : 0, + "99.950000" : 0, + "99.990000" : 0, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.00, + "bw_mean" : 0.00, + "bw_dev" : 0.00 + }, + "trim" : { + "io_bytes" : 0, + "bw" : 0, + "iops" : 0, + "runtime" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00, + "percentile" : { + "1.000000" : 0, + "5.000000" : 0, + "10.000000" : 0, + "20.000000" : 0, + "30.000000" : 0, + "40.000000" : 0, + "50.000000" : 0, + "60.000000" : 0, + "70.000000" : 0, + "80.000000" : 0, + "90.000000" : 0, + "95.000000" : 0, + "99.000000" : 0, + "99.500000" : 0, + "99.900000" : 0, + "99.950000" : 0, + "99.990000" : 0, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.00, + "bw_mean" : 0.00, + "bw_dev" : 0.00 + }, + "usr_cpu" : 0.97, + "sys_cpu" : 23.87, + "ctx" : 11121, + "majf" : 0, + "minf" : 5, + "iodepth_level" : { + "1" : 100.00, + "2" : 0.00, + "4" : 0.00, + "8" : 0.00, + "16" : 0.00, + "32" : 0.00, + ">=64" : 0.00 + }, + "latency_us" : { + "2" : 0.00, + "4" : 0.00, + "10" : 0.00, + "20" : 0.00, + "50" : 0.00, + "100" : 1.66, + "250" : 66.86, + "500" : 30.38, + "750" : 0.99, + "1000" : 0.04 + }, + "latency_ms" : { + "2" : 0.08, + "4" : 0.00, + "10" : 0.00, + "20" : 0.00, + "50" : 0.00, + "100" : 0.00, + "250" : 0.00, + "500" : 0.00, + "750" : 0.00, + "1000" : 0.00, + "2000" : 0.00, + ">=2000" : 0.00 + }, + "latency_depth" : 2, + "latency_target" : 0, + "latency_percentile" : 100.00, + "latency_window" : 0 + }, + { + "jobname" : "random-read", + "groupid" : 0, + "error" : 0, + "read" : { + "io_bytes" : 44820, + "bw" : 21, + "iops" : 5, + "runtime" : 2126510, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 53, + "max" : 5135, + "mean" : 255.20, + "stddev" : 85.82, + "percentile" : { + "1.000000" : 80, + "5.000000" : 181, + "10.000000" : 199, + "20.000000" : 241, + "30.000000" : 241, + "40.000000" : 243, + "50.000000" : 243, + "60.000000" : 245, + "70.000000" : 251, + "80.000000" : 270, + "90.000000" : 322, + "95.000000" : 378, + "99.000000" : 510, + "99.500000" : 596, + "99.900000" : 844, + "99.950000" : 1208, + "99.990000" : 1704, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 53, + "max" : 5136, + "mean" : 255.42, + "stddev" : 85.85 + }, + "bw_min" : 15176, + "bw_max" : 15712, + "bw_agg" : 100.00, + "bw_mean" : 15433.60, + "bw_dev" : 225.52 + }, + "write" : { + "io_bytes" : 0, + "bw" : 0, + "iops" : 0, + "runtime" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00, + "percentile" : { + "1.000000" : 0, + "5.000000" : 0, + "10.000000" : 0, + "20.000000" : 0, + "30.000000" : 0, + "40.000000" : 0, + "50.000000" : 0, + "60.000000" : 0, + "70.000000" : 0, + "80.000000" : 0, + "90.000000" : 0, + "95.000000" : 0, + "99.000000" : 0, + "99.500000" : 0, + "99.900000" : 0, + "99.950000" : 0, + "99.990000" : 0, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.00, + "bw_mean" : 0.00, + "bw_dev" : 0.00 + }, + "trim" : { + "io_bytes" : 0, + "bw" : 0, + "iops" : 0, + "runtime" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "clat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00, + "percentile" : { + "1.000000" : 0, + "5.000000" : 0, + "10.000000" : 0, + "20.000000" : 0, + "30.000000" : 0, + "40.000000" : 0, + "50.000000" : 0, + "60.000000" : 0, + "70.000000" : 0, + "80.000000" : 0, + "90.000000" : 0, + "95.000000" : 0, + "99.000000" : 0, + "99.500000" : 0, + "99.900000" : 0, + "99.950000" : 0, + "99.990000" : 0, + "0.00" : 0, + "0.00" : 0, + "0.00" : 0 + } + }, + "lat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.00, + "bw_mean" : 0.00, + "bw_dev" : 0.00 + }, + "usr_cpu" : 1.38, + "sys_cpu" : 23.47, + "ctx" : 11076, + "majf" : 0, + "minf" : 6, + "iodepth_level" : { + "1" : 100.00, + "2" : 0.00, + "4" : 0.00, + "8" : 0.00, + "16" : 0.00, + "32" : 0.00, + ">=64" : 0.00 + }, + "latency_us" : { + "2" : 0.00, + "4" : 0.00, + "10" : 0.00, + "20" : 0.00, + "50" : 0.00, + "100" : 1.53, + "250" : 67.13, + "500" : 30.19, + "750" : 0.98, + "1000" : 0.11 + }, + "latency_ms" : { + "2" : 0.05, + "4" : 0.00, + "10" : 0.01, + "20" : 0.00, + "50" : 0.00, + "100" : 0.00, + "250" : 0.00, + "500" : 0.00, + "750" : 0.00, + "1000" : 0.00, + "2000" : 0.00, + ">=2000" : 0.00 + }, + "latency_depth" : 2, + "latency_target" : 0, + "latency_percentile" : 100.00, + "latency_window" : 0 + } + ], + "disk_util" : [ + { + "name" : "sda", + "read_ios" : 44790, + "write_ios" : 0, + "read_merges" : 0, + "write_merges" : 0, + "read_ticks" : 2712, + "write_ticks" : 0, + "in_queue" : 2672, + "util" : 82.53 + } + ] +} """ def setUp(self): @@ -82,37 +911,37 @@ class JSONToCarbonTest(unittest.TestCase): def test_single_text_element_no_prefix(self): testconv = carbon.converter.JSONToCarbon() result = testconv.convert_to_dictionary(json.loads(self.single_json_text_element)) - + self.assertEqual("value timestamp", result["key"], result["key"]) def test_single_numeric_element_no_prefix(self): testconv = carbon.converter.JSONToCarbon() result = testconv.convert_to_dictionary(json.loads(self.single_json_numeric_element)) - + self.assertEqual("123 timestamp", result["key"], result["key"]) def test_single_text_key_space_element_no_prefix(self): testconv = carbon.converter.JSONToCarbon() result = testconv.convert_to_dictionary(json.loads(self.single_json_key_with_spaces)) - + self.assertEqual("value timestamp", result["key_with_spaces"], result["key_with_spaces"]) def test_single_text_value_space_element_no_prefix(self): testconv = carbon.converter.JSONToCarbon() result = testconv.convert_to_dictionary(json.loads(self.single_json_value_with_spaces)) - + self.assertEqual("value_with_spaces timestamp", result["key"], result["key"]) def test_map_name_with_space_no_prefix(self): testconv = carbon.converter.JSONToCarbon() result = testconv.convert_to_dictionary(json.loads(self.json_map_name_with_spaces)) - + self.assertEqual("value timestamp", result["map_with_spaces.key"], result["map_with_spaces.key"]) def test_list_name_with_space_no_prefix(self): testconv = carbon.converter.JSONToCarbon() result = testconv.convert_to_dictionary(json.loads(self.json_list_name_with_spaces)) - + self.assertEqual("value timestamp", result["list_with_spaces.1.key"], result["list_with_spaces.1.key"]) -- cgit 1.2.3-korg