summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorQiLiang <liangqi1@huawei.com>2015-08-10 00:48:17 +0800
committerQiLiang <liangqi1@huawei.com>2015-08-24 11:21:04 +0800
commit84ab6e07ef9abef13b44b69d41d5886a253751de (patch)
tree689f18a5f9b661b7244e0ea32d9b2b4d051c64f7
parentc4610ed9b6e1f77befae039072d8a5ffdb9af08a (diff)
Add test result dispatcher
This patch is the initial implementation of DB result storage. Having implemented a dispathcer which enable user select different ways to store test results according to different requirements. This patch can support not only local file storage, but also remote storage by using http request(it will call common-db-api when available). Later, local DB sotrage will be supported. This patch is raw and simple, which is implemented with reference to openstack ceilometer. Any comment is welcome. JIRA: YARDSTICK-61 Change-Id: Icaf8369edfab5d05f0819eb02d5b05dc8a04d69d Signed-off-by: QiLiang <liangqi1@huawei.com>
-rw-r--r--yardstick/benchmark/runners/base.py24
-rw-r--r--yardstick/dispatcher/__init__.py12
-rw-r--r--yardstick/dispatcher/base.py38
-rw-r--r--yardstick/dispatcher/file.py53
-rw-r--r--yardstick/dispatcher/http.py60
5 files changed, 176 insertions, 11 deletions
diff --git a/yardstick/benchmark/runners/base.py b/yardstick/benchmark/runners/base.py
index 30fa07639..848322679 100644
--- a/yardstick/benchmark/runners/base.py
+++ b/yardstick/benchmark/runners/base.py
@@ -8,7 +8,6 @@
##############################################################################
import importlib
-import json
import logging
import multiprocessing
import subprocess
@@ -18,6 +17,7 @@ log = logging.getLogger(__name__)
import yardstick.common.utils as utils
from yardstick.benchmark.scenarios import base as base_scenario
+from yardstick.dispatcher.base import Base as DispatcherBase
def _output_serializer_main(filename, queue):
@@ -25,16 +25,18 @@ def _output_serializer_main(filename, queue):
Use of this process enables multiple instances of a scenario without
messing up the output file.
'''
- with open(filename, 'a+') as outfile:
- while True:
- # blocks until data becomes available
- record = queue.get()
- if record == '_TERMINATE_':
- outfile.close()
- break
- else:
- json.dump(record, outfile)
- outfile.write('\n')
+ config = {}
+ config["type"] = "File"
+ config["file_path"] = filename
+ dispatcher = DispatcherBase.get(config)
+
+ while True:
+ # blocks until data becomes available
+ record = queue.get()
+ if record == '_TERMINATE_':
+ break
+ else:
+ dispatcher.record_result_data(record)
def _single_action(seconds, command, queue):
diff --git a/yardstick/dispatcher/__init__.py b/yardstick/dispatcher/__init__.py
new file mode 100644
index 000000000..44278c1d2
--- /dev/null
+++ b/yardstick/dispatcher/__init__.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yardstick.common.utils as utils
+
+utils.import_modules_from_package("yardstick.dispatcher")
diff --git a/yardstick/dispatcher/base.py b/yardstick/dispatcher/base.py
new file mode 100644
index 000000000..28c4c1ae6
--- /dev/null
+++ b/yardstick/dispatcher/base.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import abc
+import six
+
+import yardstick.common.utils as utils
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Base(object):
+
+ def __init__(self, conf):
+ self.conf = conf
+
+ @staticmethod
+ def get_cls(dispatcher_type):
+ '''Return class of specified type.'''
+ for dispatcher in utils.itersubclasses(Base):
+ if dispatcher_type == dispatcher.__dispatcher_type__:
+ return dispatcher
+ raise RuntimeError("No such dispatcher_type %s" % dispatcher_type)
+
+ @staticmethod
+ def get(config):
+ """Returns instance of a dispatcher for dispatcher type.
+ """
+ return Base.get_cls(config["type"])(config)
+
+ @abc.abstractmethod
+ def record_result_data(self, data):
+ """Recording result data interface."""
diff --git a/yardstick/dispatcher/file.py b/yardstick/dispatcher/file.py
new file mode 100644
index 000000000..7c644f82f
--- /dev/null
+++ b/yardstick/dispatcher/file.py
@@ -0,0 +1,53 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import logging
+
+from yardstick.dispatcher.base import Base as DispatchBase
+
+
+class FileDispatcher(DispatchBase):
+ """Dispatcher class for recording data to a file.
+ """
+
+ __dispatcher_type__ = "File"
+
+ # TODO: make parameters below configurable, currently just hard coded
+ # Path of the file to record the data
+ file_path = "/tmp/yardstick.out"
+ # The max size of the file
+ max_bytes = 0
+ # The max number of the files to keep
+ backup_count = 0
+
+ def __init__(self, conf):
+ super(FileDispatcher, self).__init__(conf)
+ self.log = None
+
+ # if the directory and path are configured, then log to the file
+ if self.file_path:
+ dispatcher_logger = logging.Logger('dispatcher.file')
+ dispatcher_logger.setLevel(logging.INFO)
+ # create rotating file handler which logs result
+ rfh = logging.handlers.RotatingFileHandler(
+ self.conf.get('file_path', self.file_path),
+ maxBytes=self.max_bytes,
+ backupCount=self.backup_count,
+ encoding='utf8')
+
+ rfh.setLevel(logging.INFO)
+ # Only wanted the data to be saved in the file, not the
+ # project root logger.
+ dispatcher_logger.propagate = False
+ dispatcher_logger.addHandler(rfh)
+ self.log = dispatcher_logger
+
+ def record_result_data(self, data):
+ if self.log:
+ self.log.info(data)
diff --git a/yardstick/dispatcher/http.py b/yardstick/dispatcher/http.py
new file mode 100644
index 000000000..c3230adb2
--- /dev/null
+++ b/yardstick/dispatcher/http.py
@@ -0,0 +1,60 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import json
+import logging
+
+import requests
+
+from yardstick.dispatcher.base import Base as DispatchBase
+
+LOG = logging.getLogger(__name__)
+
+
+class HttpDispatcher(DispatchBase):
+ """Dispatcher class for posting data into a http target.
+ """
+
+ __dispatcher_type__ = "Http"
+
+ # TODO: make parameters below configurable, currently just hard coded
+ # The target where the http request will be sent.
+ target = "http://127.0.0.1:8000/results"
+ # The max time in seconds to wait for a request to timeout.
+ timeout = 5
+
+ def __init__(self, conf):
+ super(HttpDispatcher, self).__init__(conf)
+ self.headers = {'Content-type': 'application/json'}
+ self.timeout = self.timeout
+ self.target = self.target
+
+ def record_result_data(self, data):
+ if self.target == '':
+ # if the target was not set, do not do anything
+ LOG.error('Dispatcher target was not set, no data will'
+ 'be posted.')
+ return
+
+ # We may have receive only one counter on the wire
+ if not isinstance(data, list):
+ data = [data]
+
+ for result in data:
+ try:
+ LOG.debug('Message : %s' % result)
+ res = requests.post(self.target,
+ data=json.dumps(result),
+ headers=self.headers,
+ timeout=self.timeout)
+ LOG.debug('Message posting finished with status code'
+ '%d.' % res.status_code)
+ except Exception as err:
+ LOG.exception('Failed to record result data: %s',
+ err)