Add test result dispatcher

This patch is the initial implementation of DB result storage.

Having implemented a dispathcer which enable user select different ways
to store test results according to different requirements. This patch
can support not only local file storage, but also remote storage by
using http request(it will call common-db-api when available). Later,
local DB sotrage will be supported.

This patch is raw and simple, which is implemented with reference to
openstack ceilometer.

Any comment is welcome.

JIRA: YARDSTICK-61

Change-Id: Icaf8369edfab5d05f0819eb02d5b05dc8a04d69d
Signed-off-by: QiLiang <liangqi1@huawei.com>

5 files changed, 176 insertions, 11 deletions

diff --git a/yardstick/benchmark/runners/base.py b/yardstick/benchmark/runners/base.py
index 30fa07639..848322679 100644
--- a/yardstick/benchmark/runners/base.py
+++ b/yardstick/benchmark/runners/base.py
@@ -8,7 +8,6 @@
 ##############################################################################
 
 import importlib
-import json
 import logging
 import multiprocessing
 import subprocess
@@ -18,6 +17,7 @@ log = logging.getLogger(__name__)
 
 import yardstick.common.utils as utils
 from yardstick.benchmark.scenarios import base as base_scenario
+from yardstick.dispatcher.base import Base as DispatcherBase
 
 
 def _output_serializer_main(filename, queue):
@@ -25,16 +25,18 @@ def _output_serializer_main(filename, queue):
     Use of this process enables multiple instances of a scenario without
     messing up the output file.
     '''
-    with open(filename, 'a+') as outfile:
-        while True:
-            # blocks until data becomes available
-            record = queue.get()
-            if record == '_TERMINATE_':
-                outfile.close()
-                break
-            else:
-                json.dump(record, outfile)
-                outfile.write('\n')
+    config = {}
+    config["type"] = "File"
+    config["file_path"] = filename
+    dispatcher = DispatcherBase.get(config)
+
+    while True:
+        # blocks until data becomes available
+        record = queue.get()
+        if record == '_TERMINATE_':
+            break
+        else:
+            dispatcher.record_result_data(record)
 
 
 def _single_action(seconds, command, queue):
diff --git a/yardstick/dispatcher/__init__.py b/yardstick/dispatcher/__init__.py
new file mode 100644
index 000000000..44278c1d2
--- /dev/null
+++ b/yardstick/dispatcher/__init__.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yardstick.common.utils as utils
+
+utils.import_modules_from_package("yardstick.dispatcher")
diff --git a/yardstick/dispatcher/base.py b/yardstick/dispatcher/base.py
new file mode 100644
index 000000000..28c4c1ae6
--- /dev/null
+++ b/yardstick/dispatcher/base.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import abc
+import six
+
+import yardstick.common.utils as utils
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Base(object):
+
+    def __init__(self, conf):
+        self.conf = conf
+
+    @staticmethod
+    def get_cls(dispatcher_type):
+        '''Return class of specified type.'''
+        for dispatcher in utils.itersubclasses(Base):
+            if dispatcher_type == dispatcher.__dispatcher_type__:
+                return dispatcher
+        raise RuntimeError("No such dispatcher_type %s" % dispatcher_type)
+
+    @staticmethod
+    def get(config):
+        """Returns instance of a dispatcher for dispatcher type.
+        """
+        return Base.get_cls(config["type"])(config)
+
+    @abc.abstractmethod
+    def record_result_data(self, data):
+        """Recording result data interface."""
diff --git a/yardstick/dispatcher/file.py b/yardstick/dispatcher/file.py
new file mode 100644
index 000000000..7c644f82f
--- /dev/null
+++ b/yardstick/dispatcher/file.py
@@ -0,0 +1,53 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import logging
+
+from yardstick.dispatcher.base import Base as DispatchBase
+
+
+class FileDispatcher(DispatchBase):
+    """Dispatcher class for recording data to a file.
+    """
+
+    __dispatcher_type__ = "File"
+
+    # TODO: make parameters below configurable, currently just hard coded
+    # Path of the file to record the data
+    file_path = "/tmp/yardstick.out"
+    # The max size of the file
+    max_bytes = 0
+    # The max number of the files to keep
+    backup_count = 0
+
+    def __init__(self, conf):
+        super(FileDispatcher, self).__init__(conf)
+        self.log = None
+
+        # if the directory and path are configured, then log to the file
+        if self.file_path:
+            dispatcher_logger = logging.Logger('dispatcher.file')
+            dispatcher_logger.setLevel(logging.INFO)
+            # create rotating file handler which logs result
+            rfh = logging.handlers.RotatingFileHandler(
+                self.conf.get('file_path', self.file_path),
+                maxBytes=self.max_bytes,
+                backupCount=self.backup_count,
+                encoding='utf8')
+
+            rfh.setLevel(logging.INFO)
+            # Only wanted the data to be saved in the file, not the
+            # project root logger.
+            dispatcher_logger.propagate = False
+            dispatcher_logger.addHandler(rfh)
+            self.log = dispatcher_logger
+
+    def record_result_data(self, data):
+        if self.log:
+            self.log.info(data)
diff --git a/yardstick/dispatcher/http.py b/yardstick/dispatcher/http.py
new file mode 100644
index 000000000..c3230adb2
--- /dev/null
+++ b/yardstick/dispatcher/http.py
@@ -0,0 +1,60 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import json
+import logging
+
+import requests
+
+from yardstick.dispatcher.base import Base as DispatchBase
+
+LOG = logging.getLogger(__name__)
+
+
+class HttpDispatcher(DispatchBase):
+    """Dispatcher class for posting data into a http target.
+    """
+
+    __dispatcher_type__ = "Http"
+
+    # TODO: make parameters below configurable, currently just hard coded
+    # The target where the http request will be sent.
+    target = "http://127.0.0.1:8000/results"
+    # The max time in seconds to wait for a request to timeout.
+    timeout = 5
+
+    def __init__(self, conf):
+        super(HttpDispatcher, self).__init__(conf)
+        self.headers = {'Content-type': 'application/json'}
+        self.timeout = self.timeout
+        self.target = self.target
+
+    def record_result_data(self, data):
+        if self.target == '':
+            # if the target was not set, do not do anything
+            LOG.error('Dispatcher target was not set, no data will'
+                      'be posted.')
+            return
+
+        # We may have receive only one counter on the wire
+        if not isinstance(data, list):
+            data = [data]
+
+        for result in data:
+            try:
+                LOG.debug('Message : %s' % result)
+                res = requests.post(self.target,
+                                    data=json.dumps(result),
+                                    headers=self.headers,
+                                    timeout=self.timeout)
+                LOG.debug('Message posting finished with status code'
+                          '%d.' % res.status_code)
+            except Exception as err:
+                LOG.exception('Failed to record result data: %s',
+                              err)