Merge "dispathcer to transfer test result into DB"

5 files changed, 291 insertions, 0 deletions

diff --git a/utils/dispatcher/__init__.py b/utils/dispatcher/__init__.py
new file mode 100644
index 00000000..ccf4db9f
--- /dev/null
+++ b/utils/dispatcher/__init__.py
@@ -0,0 +1,22 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+# liangqi1@huawei.com matthew.lijun@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from oslo_config import cfg
+
+import utils.dispatcher.func as func
+
+func.import_modules_from_package("utils.dispatcher")
+
+CONF = cfg.CONF
+opts = [
+    cfg.StrOpt('dispatcher',
+               default='file',
+               help='Dispatcher to store data.'),
+]
+CONF.register_opts(opts)
diff --git a/utils/dispatcher/base.py b/utils/dispatcher/base.py
new file mode 100644
index 00000000..793d374e
--- /dev/null
+++ b/utils/dispatcher/base.py
@@ -0,0 +1,42 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+# liangqi1@huawei.com matthew.lijun@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import abc
+import six
+
+import utils.dispatcher.func as func
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Base(object):
+
+    def __init__(self, conf):
+        self.conf = conf
+
+    @staticmethod
+    def get_cls(dispatcher_type):
+        '''Return class of specified type.'''
+        for dispatcher in func.itersubclasses(Base):
+            if dispatcher_type == dispatcher.__dispatcher_type__:
+                return dispatcher
+        raise RuntimeError("No such dispatcher_type %s" % dispatcher_type)
+
+    @staticmethod
+    def get(config):
+        """Returns instance of a dispatcher for dispatcher type.
+        """
+        return Base.get_cls(config["type"])(config)
+
+    @abc.abstractmethod
+    def record_result_data(self, data):
+        """Recording result data interface."""
+
+    @abc.abstractmethod
+    def flush_result_data(self):
+        """Flush result data into permanent storage media interface."""
diff --git a/utils/dispatcher/file.py b/utils/dispatcher/file.py
new file mode 100644
index 00000000..6e1df828
--- /dev/null
+++ b/utils/dispatcher/file.py
@@ -0,0 +1,66 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+# liangqi1@huawei.com matthew.lijun@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import logging
+import json
+
+from oslo_config import cfg
+
+from utils.dispatcher.base import Base as DispatchBase
+
+CONF = cfg.CONF
+opts = [
+    cfg.StrOpt('file_path',
+               default='/tmp/bottlenecks.out',
+               help='Name and the location of the file to record '
+                    'data.'),
+    cfg.IntOpt('max_bytes',
+               default=0,
+               help='The max size of the file.'),
+    cfg.IntOpt('backup_count',
+               default=0,
+               help='The max number of the files to keep.'),
+]
+CONF.register_opts(opts, group="dispatcher_file")
+
+
+class FileDispatcher(DispatchBase):
+    """Dispatcher class for recording data to a file.
+    """
+
+    __dispatcher_type__ = "File"
+
+    def __init__(self, conf):
+        super(FileDispatcher, self).__init__(conf)
+        self.log = None
+
+        # if the directory and path are configured, then log to the file
+        if CONF.dispatcher_file.file_path:
+            dispatcher_logger = logging.Logger('dispatcher.file')
+            dispatcher_logger.setLevel(logging.INFO)
+            # create rotating file handler which logs result
+            rfh = logging.handlers.RotatingFileHandler(
+                self.conf.get('file_path', CONF.dispatcher_file.file_path),
+                maxBytes=CONF.dispatcher_file.max_bytes,
+                backupCount=CONF.dispatcher_file.backup_count,
+                encoding='utf8')
+
+            rfh.setLevel(logging.INFO)
+            # Only wanted the data to be saved in the file, not the
+            # project root logger.
+            dispatcher_logger.propagate = False
+            dispatcher_logger.addHandler(rfh)
+            self.log = dispatcher_logger
+
+    def record_result_data(self, data):
+        if self.log:
+            self.log.info(json.dumps(data))
+
+    def flush_result_data(self):
+        pass
diff --git a/utils/dispatcher/func.py b/utils/dispatcher/func.py
new file mode 100644
index 00000000..71830bf2
--- /dev/null
+++ b/utils/dispatcher/func.py
@@ -0,0 +1,70 @@
+#Copyright 2013: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# bottlenecks comment: this is a modified copy of rally/rally/common/utils.py
+
+import os
+import sys
+from oslo_utils import importutils
+
+import utils
+
+
+# Decorator for cli-args
+def cliargs(*args, **kwargs):
+    def _decorator(func):
+        func.__dict__.setdefault('arguments', []).insert(0, (args, kwargs))
+        return func
+    return _decorator
+
+
+def itersubclasses(cls, _seen=None):
+    """Generator over all subclasses of a given class in depth first order."""
+
+    if not isinstance(cls, type):
+        raise TypeError("itersubclasses must be called with "
+                        "new-style classes, not %.100r" % cls)
+    _seen = _seen or set()
+    try:
+        subs = cls.__subclasses__()
+    except TypeError:   # fails only when cls is type
+        subs = cls.__subclasses__(cls)
+    for sub in subs:
+        if sub not in _seen:
+            _seen.add(sub)
+            yield sub
+            for sub in itersubclasses(sub, _seen):
+                yield sub
+
+
+def try_append_module(name, modules):
+    if name not in modules:
+        modules[name] = importutils.import_module(name)
+
+
+def import_modules_from_package(package):
+    """Import modules from package and append into sys.modules
+
+    :param: package - Full package name. For example: rally.deploy.engines
+    """
+    path = [os.path.dirname(utils.__file__), ".."] + package.split(".")
+    path = os.path.join(*path)
+    for root, dirs, files in os.walk(path):
+        for filename in files:
+            if filename.startswith("__") or not filename.endswith(".py"):
+                continue
+            new_package = ".".join(root.split(os.sep)).split("....")[1]
+            module_name = "%s.%s" % (new_package, filename[:-3])
+            try_append_module(module_name, sys.modules)
diff --git a/utils/dispatcher/http.py b/utils/dispatcher/http.py
new file mode 100644
index 00000000..d78f77fa
--- /dev/null
+++ b/utils/dispatcher/http.py
@@ -0,0 +1,91 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+# liangqi1@huawei.com matthew.lijun@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+import json
+import logging
+import requests
+
+from oslo_config import cfg
+
+from utils.dispatcher.base import Base as DispatchBase
+
+LOG = logging.getLogger(__name__)
+
+CONF = cfg.CONF
+http_dispatcher_opts = [
+    cfg.StrOpt('target',
+               default='http://127.0.0.1:8000/results',
+               help='The target where the http request will be sent. '
+                    'If this is not set, no data will be posted. For '
+                    'example: target = http://hostname:1234/path'),
+    cfg.IntOpt('timeout',
+               default=5,
+               help='The max time in seconds to wait for a request to '
+                    'timeout.'),
+]
+
+CONF.register_opts(http_dispatcher_opts, group="dispatcher_http")
+
+
+class HttpDispatcher(DispatchBase):
+    """Dispatcher class for posting data into a http target.
+    """
+
+    __dispatcher_type__ = "Http"
+
+    def __init__(self, conf):
+        super(HttpDispatcher, self).__init__(conf)
+        self.headers = {'Content-type': 'application/json'}
+        self.timeout = CONF.dispatcher_http.timeout
+        self.target = CONF.dispatcher_http.target
+        self.raw_result = []
+        self.result = {
+            "project_name": "bottlenecks",
+            "description": "bottlenecks test cases result",
+            "pod_name": os.environ.get('POD_NAME', 'unknown'),
+            "installer": os.environ.get('INSTALLER_TYPE', 'unknown'),
+            "version": os.environ.get('BOTTLENECKS_VERSION', 'unknown')
+        }
+
+    def record_result_data(self, data):
+        self.raw_result.append(data)
+
+    def flush_result_data(self):
+        if self.target == '':
+            # if the target was not set, do not do anything
+            LOG.error('Dispatcher target was not set, no data will'
+                      'be posted.')
+            return
+
+        self.result["details"] = self.raw_result
+
+        case_name = ""
+        for v in self.raw_result:
+            if isinstance(v, dict) and "scenario_cfg" in v:
+                case_name = v["scenario_cfg"]["type"]
+                break
+        if case_name == "":
+            LOG.error('Test result : %s' % json.dumps(self.result))
+            LOG.error('The case_name cannot be found, no data will be posted.')
+            return
+
+        self.result["case_name"] = case_name
+
+        try:
+            LOG.debug('Test result : %s' % json.dumps(self.result))
+            res = requests.post(self.target,
+                                data=json.dumps(self.result),
+                                headers=self.headers,
+                                timeout=self.timeout)
+            LOG.debug('Test result posting finished with status code'
+                      ' %d.' % res.status_code)
+        except Exception as err:
+            LOG.exception('Failed to record result data: %s',
+                          err)