diff options
Diffstat (limited to 'storperf')
-rw-r--r-- | storperf/carbon/converter.py | 15 | ||||
-rw-r--r-- | storperf/carbon/emitter.py | 14 | ||||
-rw-r--r-- | storperf/fio/fio_invoker.py | 76 | ||||
-rw-r--r-- | storperf/main.py | 44 |
4 files changed, 79 insertions, 70 deletions
diff --git a/storperf/carbon/converter.py b/storperf/carbon/converter.py index 4ad5be2..42dbeef 100644 --- a/storperf/carbon/converter.py +++ b/storperf/carbon/converter.py @@ -6,13 +6,22 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import calendar +import logging +import time class JSONToCarbon(object): def __init__(self): - pass - + self.logger = logging.getLogger(__name__) + def convert_to_dictionary(self, json_object, prefix=None): - self.timestamp = str(json_object['timestamp']) + # Use the timestamp reported by fio, or current time if + # not present. + if 'timestamp' in json_object: + self.timestamp = str(json_object['timestamp']) + else: + self.timestamp = str(calendar.timegm(time.gmtime())) + self.flat_dictionary = {} self.resurse_to_flat_dictionary(json_object, prefix) return self.flat_dictionary diff --git a/storperf/carbon/emitter.py b/storperf/carbon/emitter.py index 1a3f89b..526a96f 100644 --- a/storperf/carbon/emitter.py +++ b/storperf/carbon/emitter.py @@ -6,23 +6,25 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import logging import socket class CarbonMetricTransmitter(): - + carbon_host = '127.0.0.1' carbon_port = 2003 - + def __init__(self): - pass - + self.logger = logging.getLogger(__name__) + def transmit_metrics(self, metrics): self.carbon_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.carbon_socket.connect((self.carbon_host, self.carbon_port)) - + for key, metric in metrics.items(): message = key + " " + metric + "\n" + print message self.carbon_socket.send(message) - + self.carbon_socket.close() diff --git a/storperf/fio/fio_invoker.py b/storperf/fio/fio_invoker.py index 91ef6e8..722b051 100644 --- a/storperf/fio/fio_invoker.py +++ b/storperf/fio/fio_invoker.py @@ -13,34 +13,41 @@ import subprocess import json from threading import Thread +import logging + class Usage(Exception): def __init__(self, msg): self.msg = msg - + class FIOInvoker(object): def __init__(self): + self.logger = logging.getLogger(__name__) self.event_listeners = set() - + def register(self, event_listener): self.event_listeners.add(event_listener) def unregister(self, event_listener): self.event_listeners.discard(event_listener) - + def stdout_handler(self): self.json_body = "" for line in iter(self.fio_process.stdout.readline, b''): + if line.startswith("fio"): + line = "" + continue self.json_body += line try: - json_metric = json.loads(self.json_body) - self.json_body = "" - - for event_listener in self.event_listeners: - event_listener(json_metric) - - except: - if self.json_body.startswith("fio"): + if line == "}\n": + self.logger.debug("Have a json snippet: %s", self.json_body) + json_metric = json.loads(self.json_body) self.json_body = "" + + for event_listener in self.event_listeners: + event_listener(json_metric) + + except Exception, e: + self.logger.error("Error parsing JSON: %s", e) pass self.fio_process.stdout.close() @@ -50,56 +57,17 @@ class FIOInvoker(object): print line self.fio_process.stderr.close() - - def execute(self, args=[]): + + def execute(self, args=[]): self.fio_process = subprocess.Popen(['fio']+args, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE); - + t = Thread(target=self.stdout_handler, args=()) t.daemon = False t.start() - + t = Thread(target=self.stderr_handler, args=()) t.daemon = False t.start() - # fio --rw=randread --size=32m --directory=/tmp/fio-testing/data --ioengine=libaio --iodepth=2 --direct=1 --invalidate=1 --numjobs=4 --name=random-read - -def event(json_metric): - print json_metric - - -def main(argv=None): - if argv is None: - argv = sys.argv - try: - try: - opts = getopt.getopt(argv[1:], "h", ["help"]) - except getopt.error, msg: - raise Usage(msg) - - except Usage, err: - print >> sys.stderr, err.msg - print >> sys.stderr, "for help use --help" - return 2 - - for o in opts: - if o in ("-h", "--help"): - print __doc__ - return 0 - - simple_args = ['--rw=randread', '--size=32m', - '--directory=.', - '--iodepth=2', - '--direct=1', '--invalidate=1', '--numjobs=4', - '--name=random-read', '--output-format=json', - '--status-interval=3', - '--time_based', '--runtime=6'] - - invoker = FIOInvoker() - invoker.register(event) - invoker.execute(simple_args) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/storperf/main.py b/storperf/main.py index 4c5a403..f5e5581 100644 --- a/storperf/main.py +++ b/storperf/main.py @@ -7,6 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import getopt +import json +import logging +import logging.config +import os import socket import sys @@ -28,7 +32,29 @@ def event(metric): carbon_metrics = metrics_converter.convert_to_dictionary(metric, prefix) metrics_emitter.transmit_metrics(carbon_metrics) +def setup_logging( + default_path='logging.json', + default_level=logging.INFO, + env_key='LOG_CFG' +): + """Setup logging configuration + + """ + path = default_path + value = os.getenv(env_key, None) + if value: + path = value + if os.path.exists(path): + with open(path, 'rt') as f: + config = json.load(f) + logging.config.dictConfig(config) + else: + logging.basicConfig(level=default_level) + def main(argv=None): + + setup_logging() + if argv is None: argv = sys.argv try: @@ -36,28 +62,32 @@ def main(argv=None): opts = getopt.getopt(argv[1:], "h", ["help"]) except getopt.error, msg: raise Usage(msg) - + except Usage, err: print >> sys.stderr, err.msg print >> sys.stderr, "for help use --help" return 2 - + for o in opts: if o in ("-h", "--help"): print __doc__ return 0 - - simple_args = ['--rw=randread', '--size=32m', + + simple_args = ['--rw=randread', '--size=32m', '--directory=.', - '--iodepth=2', + '--iodepth=2', '--direct=1', '--invalidate=1', '--numjobs=4', '--name=random-read', '--output-format=json', - '--status-interval=3', - '--time_based', '--runtime=60'] + '--status-interval=60', + '--time_based', '--runtime=6000'] invoker = FIOInvoker() invoker.register(event) invoker.execute(simple_args) + + +logging.config.fileConfig('logging.ini') + if __name__ == "__main__": sys.exit(main()) |