From 002920e29d7fa4a28abec96773b470c90bafe55d Mon Sep 17 00:00:00 2001 From: mbeierl Date: Mon, 23 Nov 2015 08:23:47 -0800 Subject: Adding workload modules Adding the ablity to define workloads in modules which can be referenced from the API. Breaking out the test execution into its own class so it will be easier to support ReST or other interfaces. Added flake8 and code coverage reports where possible to merge and verify jobs Change-Id: Ieb51e4e7e1e989288a6f81f4757709669914a196 JIRA: STORPERF-21 Signed-off-by: mbeierl --- .pydevproject | 2 +- ci/merge.sh | 10 +- ci/verify.sh | 6 +- storperf/carbon/converter.py | 19 +- storperf/carbon/emitter.py | 14 +- storperf/fio/fio_invoker.py | 22 +- storperf/main.py | 75 +- storperf/test_executor.py | 137 +++ storperf/tests/carbon_tests/__init__.py | 0 storperf/tests/carbon_tests/emitter_test.py | 68 ++ storperf/tests/carbon_tests/json_to_carbon_test.py | 118 +++ storperf/workloads/__init__.py | 0 storperf/workloads/_base_workload.py | 54 ++ storperf/workloads/_ssd_preconditioning.py | 16 + storperf/workloads/_warm_up.py | 17 + storperf/workloads/rr.py | 16 + storperf/workloads/rs.py | 16 + storperf/workloads/rw.py | 18 + storperf/workloads/wr.py | 16 + storperf/workloads/ws.py | 16 + tests/__init__.py | 0 tests/carbon/__init__.py | 0 tests/carbon/emitter_test.py | 65 -- tests/carbon/json_to_carbon_test.py | 949 --------------------- 24 files changed, 586 insertions(+), 1068 deletions(-) create mode 100644 storperf/test_executor.py create mode 100644 storperf/tests/carbon_tests/__init__.py create mode 100644 storperf/tests/carbon_tests/emitter_test.py create mode 100644 storperf/tests/carbon_tests/json_to_carbon_test.py create mode 100644 storperf/workloads/__init__.py create mode 100644 storperf/workloads/_base_workload.py create mode 100644 storperf/workloads/_ssd_preconditioning.py create mode 100644 storperf/workloads/_warm_up.py create mode 100644 storperf/workloads/rr.py create mode 100644 storperf/workloads/rs.py create mode 100644 storperf/workloads/rw.py create mode 100644 storperf/workloads/wr.py create mode 100644 storperf/workloads/ws.py delete mode 100644 tests/__init__.py delete mode 100644 tests/carbon/__init__.py delete mode 100644 tests/carbon/emitter_test.py delete mode 100644 tests/carbon/json_to_carbon_test.py diff --git a/.pydevproject b/.pydevproject index 18c7b8a..b894d39 100644 --- a/.pydevproject +++ b/.pydevproject @@ -4,6 +4,6 @@ python 2.7 /${PROJECT_DIR_NAME}/storperf -/${PROJECT_DIR_NAME}/tests +/${PROJECT_DIR_NAME}/storperf/tests diff --git a/ci/merge.sh b/ci/merge.sh index bd1d3fe..337fc55 100755 --- a/ci/merge.sh +++ b/ci/merge.sh @@ -8,4 +8,12 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -PYTHONPATH="`pwd`/storperf":"`pwd`/tests" nosetests --with-xunit . +if [ -x /usr/bin/flake8 ]; then + flake8 storperf +fi + +nosetests --with-xunit \ + --with-coverage \ + --cover-package=storperf\ + --cover-xml \ + storperf diff --git a/ci/verify.sh b/ci/verify.sh index 333a05c..d1b0691 100755 --- a/ci/verify.sh +++ b/ci/verify.sh @@ -8,4 +8,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -exit 0 +echo "Verifying code format and compliance..." + +if [ -x /usr/bin/flake8 ] ; then + flake8 storperf +fi diff --git a/storperf/carbon/converter.py b/storperf/carbon/converter.py index 42dbeef..d551822 100644 --- a/storperf/carbon/converter.py +++ b/storperf/carbon/converter.py @@ -10,7 +10,9 @@ import calendar import logging import time + class JSONToCarbon(object): + def __init__(self): self.logger = logging.getLogger(__name__) @@ -18,11 +20,13 @@ class JSONToCarbon(object): # Use the timestamp reported by fio, or current time if # not present. if 'timestamp' in json_object: - self.timestamp = str(json_object['timestamp']) + timestamp = str(json_object.pop('timestamp')) else: - self.timestamp = str(calendar.timegm(time.gmtime())) + timestamp = str(calendar.timegm(time.gmtime())) self.flat_dictionary = {} + self.flat_dictionary['timestamp'] = timestamp + self.resurse_to_flat_dictionary(json_object, prefix) return self.flat_dictionary @@ -36,19 +40,18 @@ class JSONToCarbon(object): if hasattr(v, '__iter__'): self.resurse_to_flat_dictionary(v, key) else: - self.flat_dictionary[key] = str(v).replace(" ", "_") + " " + self.timestamp + self.flat_dictionary[key] = str(v).replace(" ", "_") elif type(json) == list: index = 0 for v in json: index += 1 if hasattr(v, '__iter__'): - self.resurse_to_flat_dictionary(v, prefix+"."+str(index)) + self.resurse_to_flat_dictionary( + v, prefix + "." + str(index)) else: if prefix is None: self.flat_dictionary[index] = str(v).replace(" ", "_") + " " + self.timestamp else: - key = prefix + "." + index; - self.flat_dictionary[key] = str(v).replace(" ", "_") + " " + self.timestamp - else: - self.flat_dictionary[json] = self.timestamp + key = prefix + "." + index + self.flat_dictionary[key] = str(v).replace(" ", "_") diff --git a/storperf/carbon/emitter.py b/storperf/carbon/emitter.py index 526a96f..e949238 100644 --- a/storperf/carbon/emitter.py +++ b/storperf/carbon/emitter.py @@ -8,7 +8,10 @@ ############################################################################## import logging +import calendar import socket +import time + class CarbonMetricTransmitter(): @@ -19,12 +22,17 @@ class CarbonMetricTransmitter(): self.logger = logging.getLogger(__name__) def transmit_metrics(self, metrics): + if 'timestamp' in metrics: + timestamp = metrics.pop('timestamp') + else: + timestamp = str(calendar.timegm(time.gmtime())) + self.carbon_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.carbon_socket.connect((self.carbon_host, self.carbon_port)) for key, metric in metrics.items(): - message = key + " " + metric + "\n" - print message - self.carbon_socket.send(message) + message = key + " " + metric + " " + timestamp + self.logger.debug("Metric: " + message) + self.carbon_socket.send(message + '\n') self.carbon_socket.close() diff --git a/storperf/fio/fio_invoker.py b/storperf/fio/fio_invoker.py index 722b051..be1b37c 100644 --- a/storperf/fio/fio_invoker.py +++ b/storperf/fio/fio_invoker.py @@ -7,19 +7,15 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -import sys -import getopt import subprocess import json -from threading import Thread +from threading import Thread import logging -class Usage(Exception): - def __init__(self, msg): - self.msg = msg class FIOInvoker(object): + def __init__(self): self.logger = logging.getLogger(__name__) self.event_listeners = set() @@ -39,7 +35,8 @@ class FIOInvoker(object): self.json_body += line try: if line == "}\n": - self.logger.debug("Have a json snippet: %s", self.json_body) + self.logger.debug( + "Have a json snippet: %s", self.json_body) json_metric = json.loads(self.json_body) self.json_body = "" @@ -59,9 +56,13 @@ class FIOInvoker(object): self.fio_process.stderr.close() def execute(self, args=[]): - self.fio_process = subprocess.Popen(['fio']+args, - universal_newlines=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE); + for arg in args: + self.logger.debug("FIO arg: " + arg) + + self.fio_process = subprocess.Popen(['fio'] + args, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) t = Thread(target=self.stdout_handler, args=()) t.daemon = False @@ -71,3 +72,4 @@ class FIOInvoker(object): t.daemon = False t.start() + t.join() diff --git a/storperf/main.py b/storperf/main.py index a40d656..2423b99 100644 --- a/storperf/main.py +++ b/storperf/main.py @@ -8,32 +8,24 @@ ############################################################################## import getopt import json -import logging import logging.config import os -import socket import sys -from fio.fio_invoker import FIOInvoker -from carbon.emitter import CarbonMetricTransmitter -from carbon.converter import JSONToCarbon +from test_executor import TestExecutor, UnknownWorkload """ """ + class Usage(Exception): + def __init__(self, msg): self.msg = msg -def event(metric): - metrics_converter = JSONToCarbon() - metrics_emitter = CarbonMetricTransmitter() - prefix = socket.getfqdn() - carbon_metrics = metrics_converter.convert_to_dictionary(metric, prefix) - metrics_emitter.transmit_metrics(carbon_metrics) def setup_logging( - default_path='logging.json', + default_path='storperf/logging.json', default_level=logging.INFO, env_key='LOG_CFG' ): @@ -51,39 +43,62 @@ def setup_logging( else: logging.basicConfig(level=default_level) -def main(argv=None): +def event(event_string): + logging.getLogger(__name__).info(event_string) + + +def main(argv=None): setup_logging() + test_executor = TestExecutor() + verbose = False + workloads = None if argv is None: argv = sys.argv try: try: - opts = getopt.getopt(argv[1:], "h", ["help"]) + opts, args = getopt.getopt(argv[1:], "t:w:scvh", + ["target=", + "workload=", + "nossd", + "nowarm", + "verbose", + "help", + ]) except getopt.error, msg: raise Usage(msg) + for o, a in opts: + if o in ("-h", "--help"): + print __doc__ + return 0 + elif o in ("-t", "--target"): + test_executor.filename = a + elif o in ("-v", "--verbose"): + verbose = True + elif o in ("-s", "--nossd"): + test_executor.precondition = False + elif o in ("-c", "--nowarm"): + test_executor.warm = False + elif o in ("-w", "--workload"): + workloads = a.split(",") + + test_executor.register_workloads(workloads) + except Usage, err: print >> sys.stderr, err.msg print >> sys.stderr, "for help use --help" return 2 + except UnknownWorkload, err: + print >> sys.stderr, err.msg + print >> sys.stderr, "for help use --help" + return 2 + + if (verbose): + test_executor.register(event) - for o in opts: - if o in ("-h", "--help"): - print __doc__ - return 0 - - simple_args = ['--rw=randread', '--size=32m', - '--directory=.', - '--iodepth=2', - '--direct=1', '--invalidate=1', '--numjobs=4', - '--name=random-read', '--output-format=json', - '--status-interval=60', - '--time_based', '--runtime=6000'] - - invoker = FIOInvoker() - invoker.register(event) - invoker.execute(simple_args) + test_executor.execute() if __name__ == "__main__": sys.exit(main()) diff --git a/storperf/test_executor.py b/storperf/test_executor.py new file mode 100644 index 0000000..5fb88d4 --- /dev/null +++ b/storperf/test_executor.py @@ -0,0 +1,137 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import imp +import logging +import os +import socket + +from os import listdir +from os.path import isfile, join + +from fio.fio_invoker import FIOInvoker +from carbon.emitter import CarbonMetricTransmitter +from carbon.converter import JSONToCarbon + + +class UnknownWorkload(Exception): + + def __init__(self, msg): + self.msg = msg + + +class TestExecutor(object): + + def __init__(self): + self.logger = logging.getLogger(__name__) + self.workload_modules = [] + self.filename = "storperf.dat" + self.precondition = True + self.warm = True + self.event_listeners = set() + self.metrics_converter = JSONToCarbon() + self.metrics_emitter = CarbonMetricTransmitter() + self.prefix = socket.getfqdn() + self.job_id = None + + def register(self, event_listener): + self.event_listeners.add(event_listener) + + def unregister(self, event_listener): + self.event_listeners.discard(event_listener) + + def event(self, metric): + carbon_metrics = self.metrics_converter.convert_to_dictionary( + metric, + self.prefix) + read_latency = carbon_metrics[self.prefix + ".jobs.1.read.lat.mean"] + write_latency = carbon_metrics[self.prefix + ".jobs.1.write.lat.mean"] + read_iops = carbon_metrics[self.prefix + ".jobs.1.read.iops"] + write_iops = carbon_metrics[self.prefix + ".jobs.1.write.iops"] + + message = "Average Latency us Read/Write: " + read_latency \ + + "/" + write_latency + " IOPS r/w: " + \ + read_iops + "/" + write_iops + + for event_listener in self.event_listeners: + event_listener(message) + + self.metrics_emitter.transmit_metrics(carbon_metrics) + + def register_workloads(self, workloads): + + if (workloads is None or workloads.length() == 0): + workload_dir = os.path.normpath( + os.path.join(os.path.dirname(__file__), "workloads")) + + workload_files = [ + f for f in listdir(workload_dir) + if isfile(join(workload_dir, f))] + + workloads = [] + + for filename in workload_files: + mname, ext = os.path.splitext(filename) + if (not mname.startswith('_')): + workloads.append(mname) + + if (self.warm is True): + workloads.insert(0, "_warm_up") + + if (self.precondition is True): + workloads.insert(0, "_ssd_preconditioning") + + for workload in workloads: + try: + workload_module = self.load_from_file("workloads/" + + workload + ".py") + self.logger.debug("Found: " + str(workload_module)) + if(workload_module is None): + raise UnknownWorkload("Unknown workload: " + workload) + self.workload_modules.append(workload_module) + except ImportError, err: + raise UnknownWorkload(err) + + def load_from_file(self, uri): + uri = os.path.normpath(os.path.join(os.path.dirname(__file__), uri)) + path, fname = os.path.split(uri) + mname, ext = os.path.splitext(fname) + no_ext = os.path.join(path, mname) + self.logger.debug("Looking for: " + no_ext) + if os.path.exists(no_ext + '.pyc'): + self.logger.debug("Loading compiled: " + mname + " from " + no_ext) + return imp.load_compiled(mname, no_ext + '.pyc') + if os.path.exists(no_ext + '.py'): + self.logger.debug("Compiling: " + mname + " from " + no_ext) + return imp.load_source(mname, no_ext + '.py') + return None + + def create_job_id(self): + return 1234 + + def execute(self): + if (self.job_id is None): + self.job_id = self.create_job_id() + + invoker = FIOInvoker() + invoker.register(self.event) + + for numjobs in [1, 2, 4]: + + for workload_module in self.workload_modules: + constructor = getattr(workload_module, "__name__") + constructorMethod = getattr(workload_module, constructor) + self.logger.debug( + "Found constructor: " + str(constructorMethod)) + workload = constructorMethod() + workload.filename = self.filename + workload.invoker = invoker + workload.options['iodepth'] = str(numjobs) + self.logger.info("Executing workload: " + constructor) + workload.execute() diff --git a/storperf/tests/carbon_tests/__init__.py b/storperf/tests/carbon_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/storperf/tests/carbon_tests/emitter_test.py b/storperf/tests/carbon_tests/emitter_test.py new file mode 100644 index 0000000..c26e837 --- /dev/null +++ b/storperf/tests/carbon_tests/emitter_test.py @@ -0,0 +1,68 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import unittest + +import json +import SocketServer +import threading +from time import sleep + +from carbon import converter +from carbon.emitter import CarbonMetricTransmitter + + +class MetricsHandler(SocketServer.BaseRequestHandler): + + def handle(self): + # Echo the back to the client + CarbonMetricTransmitterTest.response = self.request.recv(1024) + return + + +class MetricsServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + pass + + +class CarbonMetricTransmitterTest(unittest.TestCase): + listen_port = 0 + response = None + + def setUp(self): + + address = ('localhost', 0) + server = MetricsServer(address, MetricsHandler) + ip, self.listen_port = server.server_address + + t = threading.Thread(target=server.serve_forever) + t.setDaemon(True) + t.start() + + def test_transmit_metrics(self): + + testconv = converter.JSONToCarbon() + json_object = json.loads("""{"timestamp" : "12345", "key":"value" }""") + result = testconv.convert_to_dictionary(json_object, "host.run-name") + + emitter = CarbonMetricTransmitter() + emitter.carbon_port = self.listen_port + emitter.transmit_metrics(result) + + count = 0 + + while (CarbonMetricTransmitterTest.response is None and count < 10): + count += 1 + sleep(0.1) + + self.assertEqual("host.run-name.key value 12345\n", + CarbonMetricTransmitterTest.response, + CarbonMetricTransmitterTest.response) + +if __name__ == '__main__': + unittest.main() diff --git a/storperf/tests/carbon_tests/json_to_carbon_test.py b/storperf/tests/carbon_tests/json_to_carbon_test.py new file mode 100644 index 0000000..6d62418 --- /dev/null +++ b/storperf/tests/carbon_tests/json_to_carbon_test.py @@ -0,0 +1,118 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import unittest +import json + +from carbon.converter import JSONToCarbon + + +class JSONToCarbonTest(unittest.TestCase): + + single_json_text_element = """{ "key" : "value" }""" + single_json_numeric_element = """{ "key" : 123 }""" + single_json_key_with_spaces = """{ "key with spaces" : "value" }""" + single_json_value_with_spaces = """{ "key" : "value with spaces" }""" + json_map_name_with_spaces = \ + """{ "map with spaces" : { "key" : "value" } }""" + json_list_name_with_spaces = \ + """{ "list with spaces" : [{ "key" : "value" }] }""" + + simple_fio_json = """ +{ + "fio version" : "fio-2.2.10", + "timestamp" : 1444144664, + "time" : "Tue Oct 6 11:17:44 2015", + "jobs" : [ + { + "jobname" : "random-read", + "groupid" : 0, + "error" : 0, + "eta" : 0, + "elapsed" : 26, + "read" : { + "io_bytes" : 7116, + "bw" : 275, + "iops" : 68.99, + "runtime" : 25788, + "total_ios" : 1779, + "short_ios" : 0, + "drop_ios" : 0, + "slat" : { + "min" : 0, + "max" : 0, + "mean" : 0.00, + "stddev" : 0.00 + } + } + }] +} +""" + + def setUp(self): + pass + + def test_to_string(self): + testconv = JSONToCarbon() + json_object = json.loads(self.simple_fio_json) + result = testconv.convert_to_dictionary(json_object, "host.run-name") + self.assertEqual("7116", result[ + "host.run-name.jobs.1.read.io_bytes"], + result["host.run-name.jobs.1.read.io_bytes"]) + + def test_single_text_element_no_prefix(self): + testconv = JSONToCarbon() + result = testconv.convert_to_dictionary( + json.loads(self.single_json_text_element)) + + self.assertEqual("value", result["key"], result["key"]) + + def test_single_numeric_element_no_prefix(self): + testconv = JSONToCarbon() + result = testconv.convert_to_dictionary( + json.loads(self.single_json_numeric_element)) + + self.assertEqual("123", result["key"], result["key"]) + + def test_single_text_key_space_element_no_prefix(self): + testconv = JSONToCarbon() + result = testconv.convert_to_dictionary( + json.loads(self.single_json_key_with_spaces)) + + self.assertEqual( + "value", result["key_with_spaces"], result["key_with_spaces"]) + + def test_single_text_value_space_element_no_prefix(self): + testconv = JSONToCarbon() + result = testconv.convert_to_dictionary( + json.loads(self.single_json_value_with_spaces)) + + self.assertEqual("value_with_spaces", result["key"], result["key"]) + + def test_map_name_with_space_no_prefix(self): + testconv = JSONToCarbon() + result = testconv.convert_to_dictionary( + json.loads(self.json_map_name_with_spaces)) + + self.assertEqual( + "value", result["map_with_spaces.key"], + result["map_with_spaces.key"]) + + def test_list_name_with_space_no_prefix(self): + testconv = JSONToCarbon() + result = testconv.convert_to_dictionary( + json.loads(self.json_list_name_with_spaces)) + + self.assertEqual( + "value", result["list_with_spaces.1.key"], + result["list_with_spaces.1.key"]) + + +if __name__ == '__main__': + unittest.main() diff --git a/storperf/workloads/__init__.py b/storperf/workloads/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/storperf/workloads/_base_workload.py b/storperf/workloads/_base_workload.py new file mode 100644 index 0000000..7e13e45 --- /dev/null +++ b/storperf/workloads/_base_workload.py @@ -0,0 +1,54 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import logging + + +class _base_workload(object): + + def __init__(self): + self.logger = logging.getLogger(__name__) + self.default_filesize = "128M" + self.filename = 'storperf.dat' + self.options = { + 'ioengine': 'libaio', + 'direct': '1', + 'rw': 'read', + 'bs': '4k', + 'iodepth': '1', + 'numjobs': '1', + 'loops': '2', + 'output-format': 'json', + 'status-interval': '60' + } + self.invoker = None + + def execute(self): + args = [] + + if self.filename.startswith("/dev"): + self.options['size'] = "100%" + self.logger.debug( + "Profiling a device, using 100% of " + self.filename) + else: + self.options['size'] = self.default_filesize + self.logger.debug("Profiling a filesystem, using " + + self.default_filesize + " file") + + self.options['filename'] = self.filename + + self.setup() + + for key, value in self.options.iteritems(): + args.append('--' + key + "=" + value) + + self.invoker.execute(args) + + def setup(self): + pass diff --git a/storperf/workloads/_ssd_preconditioning.py b/storperf/workloads/_ssd_preconditioning.py new file mode 100644 index 0000000..66d5fa1 --- /dev/null +++ b/storperf/workloads/_ssd_preconditioning.py @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from workloads import _base_workload + + +class _ssd_preconditioning(_base_workload._base_workload): + + def setup(self): + self.options['name'] = 'ssd_preconditioning' + self.options['rw'] = 'write' diff --git a/storperf/workloads/_warm_up.py b/storperf/workloads/_warm_up.py new file mode 100644 index 0000000..8eaa2f1 --- /dev/null +++ b/storperf/workloads/_warm_up.py @@ -0,0 +1,17 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from workloads import _base_workload + + +class _warm_up(_base_workload._base_workload): + + def setup(self): + self.options['name'] = 'ssd_preconditioning' + self.options['rw'] = 'randwrite' + self.options['loops'] = '4' diff --git a/storperf/workloads/rr.py b/storperf/workloads/rr.py new file mode 100644 index 0000000..824974d --- /dev/null +++ b/storperf/workloads/rr.py @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from workloads import _base_workload + + +class rr(_base_workload._base_workload): + + def setup(self): + self.options['name'] = 'random_read' + self.options['rw'] = 'randread' diff --git a/storperf/workloads/rs.py b/storperf/workloads/rs.py new file mode 100644 index 0000000..92e3ce6 --- /dev/null +++ b/storperf/workloads/rs.py @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from workloads import _base_workload + + +class rs(_base_workload._base_workload): + + def setup(self): + self.options['name'] = 'sequential_read' + self.options['rw'] = 'read' diff --git a/storperf/workloads/rw.py b/storperf/workloads/rw.py new file mode 100644 index 0000000..2132a81 --- /dev/null +++ b/storperf/workloads/rw.py @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from workloads import _base_workload + + +class rw(_base_workload._base_workload): + + def setup(self): + self.options['name'] = 'random_readwrite' + self.options['rwmixread'] = '70' + self.options['rw'] = 'rw' + self.logger.debug(self.options) diff --git a/storperf/workloads/wr.py b/storperf/workloads/wr.py new file mode 100644 index 0000000..19b2c61 --- /dev/null +++ b/storperf/workloads/wr.py @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from workloads import _base_workload + + +class wr(_base_workload._base_workload): + + def setup(self): + self.options['name'] = 'random_write' + self.options['rw'] = 'randwrite' diff --git a/storperf/workloads/ws.py b/storperf/workloads/ws.py new file mode 100644 index 0000000..8ec2ebe --- /dev/null +++ b/storperf/workloads/ws.py @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from workloads import _base_workload + + +class ws(_base_workload._base_workload): + + def setup(self): + self.options['name'] = 'sequential_write' + self.options['rw'] = 'write' diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/carbon/__init__.py b/tests/carbon/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/carbon/emitter_test.py b/tests/carbon/emitter_test.py deleted file mode 100644 index 5caba61..0000000 --- a/tests/carbon/emitter_test.py +++ /dev/null @@ -1,65 +0,0 @@ -############################################################################## -# Copyright (c) 2015 EMC and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import unittest - -import json -import SocketServer -import threading -from time import sleep - -import carbon.converter -import carbon.emitter - -class MetricsHandler(SocketServer.BaseRequestHandler): - - def handle(self): - # Echo the back to the client - CarbonMetricTransmitterTest.response = self.request.recv(1024) - return - -class MetricsServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): - pass - -class CarbonMetricTransmitterTest(unittest.TestCase): - listen_port = 0 - response = None - - def setUp(self): - - address = ('localhost', 0) - server = MetricsServer(address, MetricsHandler) - ip, self.listen_port = server.server_address - - t = threading.Thread(target=server.serve_forever) - t.setDaemon(True) - t.start() - - def test_transmit_metrics(self): - - testconv = carbon.converter.JSONToCarbon() - json_object = json.loads("""{"timestamp" : "12345"}""") - result = testconv.convert_to_dictionary(json_object, "host.run-name") - - emitter = carbon.emitter.CarbonMetricTransmitter() - emitter.carbon_port = self.listen_port - emitter.transmit_metrics(result) - - count = 0 - - while (CarbonMetricTransmitterTest.response == None and count < 10): - count += 1 - sleep(0.1) - - self.assertEqual("host.run-name.timestamp 12345 12345\n", - CarbonMetricTransmitterTest.response, - CarbonMetricTransmitterTest.response) - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/tests/carbon/json_to_carbon_test.py b/tests/carbon/json_to_carbon_test.py deleted file mode 100644 index 060d93d..0000000 --- a/tests/carbon/json_to_carbon_test.py +++ /dev/null @@ -1,949 +0,0 @@ -############################################################################## -# Copyright (c) 2015 EMC and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import unittest - -import json - -import carbon.converter - -class JSONToCarbonTest(unittest.TestCase): - - single_json_text_element = """{ "timestamp" : "timestamp", "key" : "value" }""" - single_json_numeric_element = """{ "timestamp" : "timestamp", "key" : 123 }""" - single_json_key_with_spaces = """{ "timestamp" : "timestamp", "key with spaces" : "value" }""" - single_json_value_with_spaces = """{ "timestamp" : "timestamp", "key" : "value with spaces" }""" - json_map_name_with_spaces = """{ "timestamp" : "timestamp", "map with spaces" : { "key" : "value" } }""" - json_list_name_with_spaces = """{ "timestamp" : "timestamp", "list with spaces" : [{ "key" : "value" }] }""" - - sample_json = """ -{ - "colorsArray":[{ - "colorName":"red", - "hexValue":"#f00" - }, - { - "colorName":"green", - "hexValue":"#0f0" - }, - { - "colorName":"blue", - "hexValue":"#00f" - } - ] -}""" - - simple_fio_json = """ -{ - "fio version" : "fio-2.2.10", - "timestamp" : 1444144664, - "time" : "Tue Oct 6 11:17:44 2015", - "jobs" : [ - { - "jobname" : "random-read", - "groupid" : 0, - "error" : 0, - "eta" : 0, - "elapsed" : 26, - "read" : { - "io_bytes" : 7116, - "bw" : 275, - "iops" : 68.99, - "runtime" : 25788, - "total_ios" : 1779, - "short_ios" : 0, - "drop_ios" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - } - } - }] -} -""" - - full_fio = """ -{ - "fio version" : "fio-2.1.11", - "jobs" : [ - { - "jobname" : "random-read", - "groupid" : 0, - "error" : 0, - "read" : { - "io_bytes" : 44920, - "bw" : 21, - "iops" : 5, - "runtime" : 2127550, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 53, - "max" : 6329, - "mean" : 255.34, - "stddev" : 94.40, - "percentile" : { - "1.000000" : 84, - "5.000000" : 181, - "10.000000" : 203, - "20.000000" : 241, - "30.000000" : 241, - "40.000000" : 243, - "50.000000" : 243, - "60.000000" : 243, - "70.000000" : 251, - "80.000000" : 270, - "90.000000" : 318, - "95.000000" : 370, - "99.000000" : 506, - "99.500000" : 596, - "99.900000" : 812, - "99.950000" : 1032, - "99.990000" : 2992, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 53, - "max" : 6330, - "mean" : 255.56, - "stddev" : 94.41 - }, - "bw_min" : 15008, - "bw_max" : 15664, - "bw_agg" : 100.00, - "bw_mean" : 15433.60, - "bw_dev" : 264.57 - }, - "write" : { - "io_bytes" : 0, - "bw" : 0, - "iops" : 0, - "runtime" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00, - "percentile" : { - "1.000000" : 0, - "5.000000" : 0, - "10.000000" : 0, - "20.000000" : 0, - "30.000000" : 0, - "40.000000" : 0, - "50.000000" : 0, - "60.000000" : 0, - "70.000000" : 0, - "80.000000" : 0, - "90.000000" : 0, - "95.000000" : 0, - "99.000000" : 0, - "99.500000" : 0, - "99.900000" : 0, - "99.950000" : 0, - "99.990000" : 0, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.00, - "bw_mean" : 0.00, - "bw_dev" : 0.00 - }, - "trim" : { - "io_bytes" : 0, - "bw" : 0, - "iops" : 0, - "runtime" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00, - "percentile" : { - "1.000000" : 0, - "5.000000" : 0, - "10.000000" : 0, - "20.000000" : 0, - "30.000000" : 0, - "40.000000" : 0, - "50.000000" : 0, - "60.000000" : 0, - "70.000000" : 0, - "80.000000" : 0, - "90.000000" : 0, - "95.000000" : 0, - "99.000000" : 0, - "99.500000" : 0, - "99.900000" : 0, - "99.950000" : 0, - "99.990000" : 0, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.00, - "bw_mean" : 0.00, - "bw_dev" : 0.00 - }, - "usr_cpu" : 0.96, - "sys_cpu" : 23.82, - "ctx" : 11122, - "majf" : 0, - "minf" : 6, - "iodepth_level" : { - "1" : 100.00, - "2" : 0.00, - "4" : 0.00, - "8" : 0.00, - "16" : 0.00, - "32" : 0.00, - ">=64" : 0.00 - }, - "latency_us" : { - "2" : 0.00, - "4" : 0.00, - "10" : 0.00, - "20" : 0.00, - "50" : 0.00, - "100" : 1.31, - "250" : 67.24, - "500" : 30.34, - "750" : 0.94, - "1000" : 0.12 - }, - "latency_ms" : { - "2" : 0.03, - "4" : 0.02, - "10" : 0.01, - "20" : 0.00, - "50" : 0.00, - "100" : 0.00, - "250" : 0.00, - "500" : 0.00, - "750" : 0.00, - "1000" : 0.00, - "2000" : 0.00, - ">=2000" : 0.00 - }, - "latency_depth" : 2, - "latency_target" : 0, - "latency_percentile" : 100.00, - "latency_window" : 0 - }, - { - "jobname" : "random-read", - "groupid" : 0, - "error" : 0, - "read" : { - "io_bytes" : 44992, - "bw" : 21, - "iops" : 5, - "runtime" : 2119748, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 53, - "max" : 2976, - "mean" : 254.65, - "stddev" : 81.49, - "percentile" : { - "1.000000" : 79, - "5.000000" : 181, - "10.000000" : 195, - "20.000000" : 241, - "30.000000" : 241, - "40.000000" : 243, - "50.000000" : 243, - "60.000000" : 243, - "70.000000" : 251, - "80.000000" : 270, - "90.000000" : 318, - "95.000000" : 374, - "99.000000" : 510, - "99.500000" : 564, - "99.900000" : 972, - "99.950000" : 1496, - "99.990000" : 2800, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 53, - "max" : 2977, - "mean" : 254.87, - "stddev" : 81.50 - }, - "bw_min" : 15144, - "bw_max" : 15704, - "bw_agg" : 100.00, - "bw_mean" : 15467.20, - "bw_dev" : 222.90 - }, - "write" : { - "io_bytes" : 0, - "bw" : 0, - "iops" : 0, - "runtime" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00, - "percentile" : { - "1.000000" : 0, - "5.000000" : 0, - "10.000000" : 0, - "20.000000" : 0, - "30.000000" : 0, - "40.000000" : 0, - "50.000000" : 0, - "60.000000" : 0, - "70.000000" : 0, - "80.000000" : 0, - "90.000000" : 0, - "95.000000" : 0, - "99.000000" : 0, - "99.500000" : 0, - "99.900000" : 0, - "99.950000" : 0, - "99.990000" : 0, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.00, - "bw_mean" : 0.00, - "bw_dev" : 0.00 - }, - "trim" : { - "io_bytes" : 0, - "bw" : 0, - "iops" : 0, - "runtime" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00, - "percentile" : { - "1.000000" : 0, - "5.000000" : 0, - "10.000000" : 0, - "20.000000" : 0, - "30.000000" : 0, - "40.000000" : 0, - "50.000000" : 0, - "60.000000" : 0, - "70.000000" : 0, - "80.000000" : 0, - "90.000000" : 0, - "95.000000" : 0, - "99.000000" : 0, - "99.500000" : 0, - "99.900000" : 0, - "99.950000" : 0, - "99.990000" : 0, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.00, - "bw_mean" : 0.00, - "bw_dev" : 0.00 - }, - "usr_cpu" : 2.07, - "sys_cpu" : 22.74, - "ctx" : 11126, - "majf" : 0, - "minf" : 7, - "iodepth_level" : { - "1" : 100.00, - "2" : 0.00, - "4" : 0.00, - "8" : 0.00, - "16" : 0.00, - "32" : 0.00, - ">=64" : 0.00 - }, - "latency_us" : { - "2" : 0.00, - "4" : 0.00, - "10" : 0.00, - "20" : 0.00, - "50" : 0.00, - "100" : 1.53, - "250" : 67.31, - "500" : 30.06, - "750" : 0.90, - "1000" : 0.12 - }, - "latency_ms" : { - "2" : 0.07, - "4" : 0.02, - "10" : 0.00, - "20" : 0.00, - "50" : 0.00, - "100" : 0.00, - "250" : 0.00, - "500" : 0.00, - "750" : 0.00, - "1000" : 0.00, - "2000" : 0.00, - ">=2000" : 0.00 - }, - "latency_depth" : 2, - "latency_target" : 0, - "latency_percentile" : 100.00, - "latency_window" : 0 - }, - { - "jobname" : "random-read", - "groupid" : 0, - "error" : 0, - "read" : { - "io_bytes" : 45024, - "bw" : 21, - "iops" : 5, - "runtime" : 2114297, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 53, - "max" : 1752, - "mean" : 254.25, - "stddev" : 72.36, - "percentile" : { - "1.000000" : 76, - "5.000000" : 181, - "10.000000" : 197, - "20.000000" : 241, - "30.000000" : 241, - "40.000000" : 243, - "50.000000" : 243, - "60.000000" : 245, - "70.000000" : 251, - "80.000000" : 270, - "90.000000" : 318, - "95.000000" : 374, - "99.000000" : 510, - "99.500000" : 564, - "99.900000" : 844, - "99.950000" : 1176, - "99.990000" : 1720, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 53, - "max" : 1752, - "mean" : 254.48, - "stddev" : 72.40 - }, - "bw_min" : 15320, - "bw_max" : 15784, - "bw_agg" : 100.00, - "bw_mean" : 15504.00, - "bw_dev" : 191.33 - }, - "write" : { - "io_bytes" : 0, - "bw" : 0, - "iops" : 0, - "runtime" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00, - "percentile" : { - "1.000000" : 0, - "5.000000" : 0, - "10.000000" : 0, - "20.000000" : 0, - "30.000000" : 0, - "40.000000" : 0, - "50.000000" : 0, - "60.000000" : 0, - "70.000000" : 0, - "80.000000" : 0, - "90.000000" : 0, - "95.000000" : 0, - "99.000000" : 0, - "99.500000" : 0, - "99.900000" : 0, - "99.950000" : 0, - "99.990000" : 0, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.00, - "bw_mean" : 0.00, - "bw_dev" : 0.00 - }, - "trim" : { - "io_bytes" : 0, - "bw" : 0, - "iops" : 0, - "runtime" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00, - "percentile" : { - "1.000000" : 0, - "5.000000" : 0, - "10.000000" : 0, - "20.000000" : 0, - "30.000000" : 0, - "40.000000" : 0, - "50.000000" : 0, - "60.000000" : 0, - "70.000000" : 0, - "80.000000" : 0, - "90.000000" : 0, - "95.000000" : 0, - "99.000000" : 0, - "99.500000" : 0, - "99.900000" : 0, - "99.950000" : 0, - "99.990000" : 0, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.00, - "bw_mean" : 0.00, - "bw_dev" : 0.00 - }, - "usr_cpu" : 0.97, - "sys_cpu" : 23.87, - "ctx" : 11121, - "majf" : 0, - "minf" : 5, - "iodepth_level" : { - "1" : 100.00, - "2" : 0.00, - "4" : 0.00, - "8" : 0.00, - "16" : 0.00, - "32" : 0.00, - ">=64" : 0.00 - }, - "latency_us" : { - "2" : 0.00, - "4" : 0.00, - "10" : 0.00, - "20" : 0.00, - "50" : 0.00, - "100" : 1.66, - "250" : 66.86, - "500" : 30.38, - "750" : 0.99, - "1000" : 0.04 - }, - "latency_ms" : { - "2" : 0.08, - "4" : 0.00, - "10" : 0.00, - "20" : 0.00, - "50" : 0.00, - "100" : 0.00, - "250" : 0.00, - "500" : 0.00, - "750" : 0.00, - "1000" : 0.00, - "2000" : 0.00, - ">=2000" : 0.00 - }, - "latency_depth" : 2, - "latency_target" : 0, - "latency_percentile" : 100.00, - "latency_window" : 0 - }, - { - "jobname" : "random-read", - "groupid" : 0, - "error" : 0, - "read" : { - "io_bytes" : 44820, - "bw" : 21, - "iops" : 5, - "runtime" : 2126510, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 53, - "max" : 5135, - "mean" : 255.20, - "stddev" : 85.82, - "percentile" : { - "1.000000" : 80, - "5.000000" : 181, - "10.000000" : 199, - "20.000000" : 241, - "30.000000" : 241, - "40.000000" : 243, - "50.000000" : 243, - "60.000000" : 245, - "70.000000" : 251, - "80.000000" : 270, - "90.000000" : 322, - "95.000000" : 378, - "99.000000" : 510, - "99.500000" : 596, - "99.900000" : 844, - "99.950000" : 1208, - "99.990000" : 1704, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 53, - "max" : 5136, - "mean" : 255.42, - "stddev" : 85.85 - }, - "bw_min" : 15176, - "bw_max" : 15712, - "bw_agg" : 100.00, - "bw_mean" : 15433.60, - "bw_dev" : 225.52 - }, - "write" : { - "io_bytes" : 0, - "bw" : 0, - "iops" : 0, - "runtime" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00, - "percentile" : { - "1.000000" : 0, - "5.000000" : 0, - "10.000000" : 0, - "20.000000" : 0, - "30.000000" : 0, - "40.000000" : 0, - "50.000000" : 0, - "60.000000" : 0, - "70.000000" : 0, - "80.000000" : 0, - "90.000000" : 0, - "95.000000" : 0, - "99.000000" : 0, - "99.500000" : 0, - "99.900000" : 0, - "99.950000" : 0, - "99.990000" : 0, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.00, - "bw_mean" : 0.00, - "bw_dev" : 0.00 - }, - "trim" : { - "io_bytes" : 0, - "bw" : 0, - "iops" : 0, - "runtime" : 0, - "slat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "clat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00, - "percentile" : { - "1.000000" : 0, - "5.000000" : 0, - "10.000000" : 0, - "20.000000" : 0, - "30.000000" : 0, - "40.000000" : 0, - "50.000000" : 0, - "60.000000" : 0, - "70.000000" : 0, - "80.000000" : 0, - "90.000000" : 0, - "95.000000" : 0, - "99.000000" : 0, - "99.500000" : 0, - "99.900000" : 0, - "99.950000" : 0, - "99.990000" : 0, - "0.00" : 0, - "0.00" : 0, - "0.00" : 0 - } - }, - "lat" : { - "min" : 0, - "max" : 0, - "mean" : 0.00, - "stddev" : 0.00 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.00, - "bw_mean" : 0.00, - "bw_dev" : 0.00 - }, - "usr_cpu" : 1.38, - "sys_cpu" : 23.47, - "ctx" : 11076, - "majf" : 0, - "minf" : 6, - "iodepth_level" : { - "1" : 100.00, - "2" : 0.00, - "4" : 0.00, - "8" : 0.00, - "16" : 0.00, - "32" : 0.00, - ">=64" : 0.00 - }, - "latency_us" : { - "2" : 0.00, - "4" : 0.00, - "10" : 0.00, - "20" : 0.00, - "50" : 0.00, - "100" : 1.53, - "250" : 67.13, - "500" : 30.19, - "750" : 0.98, - "1000" : 0.11 - }, - "latency_ms" : { - "2" : 0.05, - "4" : 0.00, - "10" : 0.01, - "20" : 0.00, - "50" : 0.00, - "100" : 0.00, - "250" : 0.00, - "500" : 0.00, - "750" : 0.00, - "1000" : 0.00, - "2000" : 0.00, - ">=2000" : 0.00 - }, - "latency_depth" : 2, - "latency_target" : 0, - "latency_percentile" : 100.00, - "latency_window" : 0 - } - ], - "disk_util" : [ - { - "name" : "sda", - "read_ios" : 44790, - "write_ios" : 0, - "read_merges" : 0, - "write_merges" : 0, - "read_ticks" : 2712, - "write_ticks" : 0, - "in_queue" : 2672, - "util" : 82.53 - } - ] -} -""" - - def setUp(self): - pass - - def test_to_string(self): - testconv = carbon.converter.JSONToCarbon() - json_object = json.loads(self.simple_fio_json) - result = testconv.convert_to_dictionary(json_object, "host.run-name") - self.assertEqual("7116 1444144664", result["host.run-name.jobs.1.read.io_bytes"], result["host.run-name.jobs.1.read.io_bytes"]) - - def test_single_text_element_no_prefix(self): - testconv = carbon.converter.JSONToCarbon() - result = testconv.convert_to_dictionary(json.loads(self.single_json_text_element)) - - self.assertEqual("value timestamp", result["key"], result["key"]) - - def test_single_numeric_element_no_prefix(self): - testconv = carbon.converter.JSONToCarbon() - result = testconv.convert_to_dictionary(json.loads(self.single_json_numeric_element)) - - self.assertEqual("123 timestamp", result["key"], result["key"]) - - def test_single_text_key_space_element_no_prefix(self): - testconv = carbon.converter.JSONToCarbon() - result = testconv.convert_to_dictionary(json.loads(self.single_json_key_with_spaces)) - - self.assertEqual("value timestamp", result["key_with_spaces"], result["key_with_spaces"]) - - def test_single_text_value_space_element_no_prefix(self): - testconv = carbon.converter.JSONToCarbon() - result = testconv.convert_to_dictionary(json.loads(self.single_json_value_with_spaces)) - - self.assertEqual("value_with_spaces timestamp", result["key"], result["key"]) - - def test_map_name_with_space_no_prefix(self): - testconv = carbon.converter.JSONToCarbon() - result = testconv.convert_to_dictionary(json.loads(self.json_map_name_with_spaces)) - - self.assertEqual("value timestamp", result["map_with_spaces.key"], result["map_with_spaces.key"]) - - def test_list_name_with_space_no_prefix(self): - testconv = carbon.converter.JSONToCarbon() - result = testconv.convert_to_dictionary(json.loads(self.json_list_name_with_spaces)) - - self.assertEqual("value timestamp", result["list_with_spaces.1.key"], result["list_with_spaces.1.key"]) - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file -- cgit 1.2.3-korg