summaryrefslogtreecommitdiffstats
path: root/storperf
diff options
context:
space:
mode:
authormbeierl <mark.beierl@emc.com>2015-10-15 10:48:58 -0400
committerMark Beierl <mark.beierl@emc.com>2015-10-15 19:08:24 +0000
commite708f56598414929091524ce68a56da602d082e7 (patch)
tree26867be9b46ce0e47af1dd236d9ba186e3e29291 /storperf
parentb0f94dcb1f28161d4ab2fb186e4867b2ba473a32 (diff)
Logging and timestamp
Added logging and fixed issue if fio does not produce the current timestamp in its output JIRA: STORPERF-4 Change-Id: Ifd0dbc4e17d984907e63089ebfae1d0e9e749dcc Signed-off-by: mbeierl <mark.beierl@emc.com>
Diffstat (limited to 'storperf')
-rw-r--r--storperf/carbon/converter.py15
-rw-r--r--storperf/carbon/emitter.py14
-rw-r--r--storperf/fio/fio_invoker.py76
-rw-r--r--storperf/main.py44
4 files changed, 79 insertions, 70 deletions
diff --git a/storperf/carbon/converter.py b/storperf/carbon/converter.py
index 4ad5be2..42dbeef 100644
--- a/storperf/carbon/converter.py
+++ b/storperf/carbon/converter.py
@@ -6,13 +6,22 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+import calendar
+import logging
+import time
class JSONToCarbon(object):
def __init__(self):
- pass
-
+ self.logger = logging.getLogger(__name__)
+
def convert_to_dictionary(self, json_object, prefix=None):
- self.timestamp = str(json_object['timestamp'])
+ # Use the timestamp reported by fio, or current time if
+ # not present.
+ if 'timestamp' in json_object:
+ self.timestamp = str(json_object['timestamp'])
+ else:
+ self.timestamp = str(calendar.timegm(time.gmtime()))
+
self.flat_dictionary = {}
self.resurse_to_flat_dictionary(json_object, prefix)
return self.flat_dictionary
diff --git a/storperf/carbon/emitter.py b/storperf/carbon/emitter.py
index 1a3f89b..526a96f 100644
--- a/storperf/carbon/emitter.py
+++ b/storperf/carbon/emitter.py
@@ -6,23 +6,25 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+import logging
import socket
class CarbonMetricTransmitter():
-
+
carbon_host = '127.0.0.1'
carbon_port = 2003
-
+
def __init__(self):
- pass
-
+ self.logger = logging.getLogger(__name__)
+
def transmit_metrics(self, metrics):
self.carbon_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.carbon_socket.connect((self.carbon_host, self.carbon_port))
-
+
for key, metric in metrics.items():
message = key + " " + metric + "\n"
+ print message
self.carbon_socket.send(message)
-
+
self.carbon_socket.close()
diff --git a/storperf/fio/fio_invoker.py b/storperf/fio/fio_invoker.py
index 91ef6e8..722b051 100644
--- a/storperf/fio/fio_invoker.py
+++ b/storperf/fio/fio_invoker.py
@@ -13,34 +13,41 @@ import subprocess
import json
from threading import Thread
+import logging
+
class Usage(Exception):
def __init__(self, msg):
self.msg = msg
-
+
class FIOInvoker(object):
def __init__(self):
+ self.logger = logging.getLogger(__name__)
self.event_listeners = set()
-
+
def register(self, event_listener):
self.event_listeners.add(event_listener)
def unregister(self, event_listener):
self.event_listeners.discard(event_listener)
-
+
def stdout_handler(self):
self.json_body = ""
for line in iter(self.fio_process.stdout.readline, b''):
+ if line.startswith("fio"):
+ line = ""
+ continue
self.json_body += line
try:
- json_metric = json.loads(self.json_body)
- self.json_body = ""
-
- for event_listener in self.event_listeners:
- event_listener(json_metric)
-
- except:
- if self.json_body.startswith("fio"):
+ if line == "}\n":
+ self.logger.debug("Have a json snippet: %s", self.json_body)
+ json_metric = json.loads(self.json_body)
self.json_body = ""
+
+ for event_listener in self.event_listeners:
+ event_listener(json_metric)
+
+ except Exception, e:
+ self.logger.error("Error parsing JSON: %s", e)
pass
self.fio_process.stdout.close()
@@ -50,56 +57,17 @@ class FIOInvoker(object):
print line
self.fio_process.stderr.close()
-
- def execute(self, args=[]):
+
+ def execute(self, args=[]):
self.fio_process = subprocess.Popen(['fio']+args,
universal_newlines=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE);
-
+
t = Thread(target=self.stdout_handler, args=())
t.daemon = False
t.start()
-
+
t = Thread(target=self.stderr_handler, args=())
t.daemon = False
t.start()
- # fio --rw=randread --size=32m --directory=/tmp/fio-testing/data --ioengine=libaio --iodepth=2 --direct=1 --invalidate=1 --numjobs=4 --name=random-read
-
-def event(json_metric):
- print json_metric
-
-
-def main(argv=None):
- if argv is None:
- argv = sys.argv
- try:
- try:
- opts = getopt.getopt(argv[1:], "h", ["help"])
- except getopt.error, msg:
- raise Usage(msg)
-
- except Usage, err:
- print >> sys.stderr, err.msg
- print >> sys.stderr, "for help use --help"
- return 2
-
- for o in opts:
- if o in ("-h", "--help"):
- print __doc__
- return 0
-
- simple_args = ['--rw=randread', '--size=32m',
- '--directory=.',
- '--iodepth=2',
- '--direct=1', '--invalidate=1', '--numjobs=4',
- '--name=random-read', '--output-format=json',
- '--status-interval=3',
- '--time_based', '--runtime=6']
-
- invoker = FIOInvoker()
- invoker.register(event)
- invoker.execute(simple_args)
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/storperf/main.py b/storperf/main.py
index 4c5a403..f5e5581 100644
--- a/storperf/main.py
+++ b/storperf/main.py
@@ -7,6 +7,10 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
import getopt
+import json
+import logging
+import logging.config
+import os
import socket
import sys
@@ -28,7 +32,29 @@ def event(metric):
carbon_metrics = metrics_converter.convert_to_dictionary(metric, prefix)
metrics_emitter.transmit_metrics(carbon_metrics)
+def setup_logging(
+ default_path='logging.json',
+ default_level=logging.INFO,
+ env_key='LOG_CFG'
+):
+ """Setup logging configuration
+
+ """
+ path = default_path
+ value = os.getenv(env_key, None)
+ if value:
+ path = value
+ if os.path.exists(path):
+ with open(path, 'rt') as f:
+ config = json.load(f)
+ logging.config.dictConfig(config)
+ else:
+ logging.basicConfig(level=default_level)
+
def main(argv=None):
+
+ setup_logging()
+
if argv is None:
argv = sys.argv
try:
@@ -36,28 +62,32 @@ def main(argv=None):
opts = getopt.getopt(argv[1:], "h", ["help"])
except getopt.error, msg:
raise Usage(msg)
-
+
except Usage, err:
print >> sys.stderr, err.msg
print >> sys.stderr, "for help use --help"
return 2
-
+
for o in opts:
if o in ("-h", "--help"):
print __doc__
return 0
-
- simple_args = ['--rw=randread', '--size=32m',
+
+ simple_args = ['--rw=randread', '--size=32m',
'--directory=.',
- '--iodepth=2',
+ '--iodepth=2',
'--direct=1', '--invalidate=1', '--numjobs=4',
'--name=random-read', '--output-format=json',
- '--status-interval=3',
- '--time_based', '--runtime=60']
+ '--status-interval=60',
+ '--time_based', '--runtime=6000']
invoker = FIOInvoker()
invoker.register(event)
invoker.execute(simple_args)
+
+
+logging.config.fileConfig('logging.ini')
+
if __name__ == "__main__":
sys.exit(main())