Support general configuration file

Use openstack library oslo_config for parsing configuration options
from the command line and configuration files.
Refer http://docs.openstack.org/developer/oslo.config/ or rally
source code for more info on oslo_config library usage.

This patch is initially for test result dispatcher configuration,
but it is very general to use.

Usage:
  0) install yardstick
  1) mkdir /etc/yardstick
  2) cp <repo_root_dir>/etc/yardstick/yardstick.conf.sample \
        /etc/yardstick/yardstick.conf
  3) edit /etc/yardstick/yardstick.conf
  4) run `yardstick task start xxx` cmd

JIRA: YARDSTICK-61

Change-Id: I01677ef6e9ab7c1975aa193799195e850da73478
Signed-off-by: QiLiang <liangqi1@huawei.com>

6 files changed, 117 insertions, 75 deletions

diff --git a/etc/yardstick/yardstick.conf.sample b/etc/yardstick/yardstick.conf.sample
new file mode 100644
index 000000000..82326dd1b
--- /dev/null
+++ b/etc/yardstick/yardstick.conf.sample
@@ -0,0 +1,13 @@
+[DEFAULT]
+# verbose = True
+# debug = True
+# dispatcher = http
+
+[dispatcher_http]
+# timeout = 5
+# target = http://127.0.0.1:8000/results
+
+[dispatcher_file]
+# file_path = /tmp/yardstick.out
+# max_bytes = 0
+# backup_count = 0
diff --git a/yardstick/benchmark/runners/base.py b/yardstick/benchmark/runners/base.py
index d8783f3c1..265e5d158 100644
--- a/yardstick/benchmark/runners/base.py
+++ b/yardstick/benchmark/runners/base.py
@@ -15,10 +15,14 @@ import time
 
 log = logging.getLogger(__name__)
 
+from oslo_config import cfg
+
 import yardstick.common.utils as utils
 from yardstick.benchmark.scenarios import base as base_scenario
 from yardstick.dispatcher.base import Base as DispatcherBase
 
+CONF = cfg.CONF
+
 
 def _output_serializer_main(filename, queue):
     '''entrypoint for the singleton subprocess writing to outfile
@@ -26,7 +30,7 @@ def _output_serializer_main(filename, queue):
     messing up the output file.
     '''
     config = {}
-    config["type"] = "File"
+    config["type"] = CONF.dispatcher.capitalize()
     config["file_path"] = filename
     dispatcher = DispatcherBase.get(config)
 
diff --git a/yardstick/cmd/cli.py b/yardstick/cmd/cli.py
index ae7f3be40..1f058a5a1 100644
--- a/yardstick/cmd/cli.py
+++ b/yardstick/cmd/cli.py
@@ -11,16 +11,45 @@
 Command-line interface to yardstick
 '''
 
-import argparse
 import logging
+import os
+import sys
 
 from pkg_resources import get_distribution
 from argparse import RawDescriptionHelpFormatter
+from oslo_config import cfg
 
 from yardstick.cmd.commands import task
 from yardstick.cmd.commands import runner
 from yardstick.cmd.commands import scenario
 
+CONF = cfg.CONF
+cli_opts = [
+    cfg.BoolOpt('debug',
+                short='d',
+                default=False,
+                help='increase output verbosity to debug'),
+    cfg.BoolOpt('verbose',
+                short='v',
+                default=False,
+                help='increase output verbosity to info')
+]
+CONF.register_cli_opts(cli_opts)
+
+CONFIG_SEARCH_PATHS = [sys.prefix + "/etc/yardstick",
+                       "~/.yardstick",
+                       "/etc/yardstick"]
+
+
+def find_config_files(path_list):
+    for path in path_list:
+        abspath = os.path.abspath(os.path.expanduser(path))
+        confname = abspath + "/yardstick.conf"
+        if os.path.isfile(confname):
+            return [confname]
+
+    return None
+
 
 class YardstickCLI():
     '''Command-line interface to yardstick'''
@@ -36,38 +65,6 @@ class YardstickCLI():
         self._version = 'yardstick version %s ' % \
             get_distribution('yardstick').version
 
-    def _get_base_parser(self):
-        '''get base (top level) parser'''
-
-        parser = argparse.ArgumentParser(
-            prog='yardstick',
-            formatter_class=RawDescriptionHelpFormatter,
-            description=YardstickCLI.__doc__ or ''
-        )
-
-        # Global options
-
-        parser.add_argument(
-            "-V", "--version",
-            help="display version",
-            version=self._version,
-            action="version"
-        )
-
-        parser.add_argument(
-            "-d", "--debug",
-            help="increase output verbosity to debug",
-            action="store_true"
-        )
-
-        parser.add_argument(
-            "-v", "--verbose",
-            help="increase output verbosity to info",
-            action="store_true"
-        )
-
-        return parser
-
     def _find_actions(self, subparsers, actions_module):
         '''find action methods'''
         # Find action methods inside actions_module and
@@ -87,17 +84,10 @@ class YardstickCLI():
                 subparser.add_argument(*args, **kwargs)
             subparser.set_defaults(func=callback)
 
-    def _get_parser(self):
-        '''get a command-line parser'''
-        parser = self._get_base_parser()
-
-        subparsers = parser.add_subparsers(
-            title='subcommands',
-        )
-
-        # add subcommands
-        for category in YardstickCLI.categories:
-            command_object = YardstickCLI.categories[category]()
+    def _add_command_parsers(self, categories, subparsers):
+        '''add commands to command-line parser'''
+        for category in categories:
+            command_object = categories[category]()
             desc = command_object.__doc__ or ''
             subparser = subparsers.add_parser(
                 category, description=desc,
@@ -107,26 +97,32 @@ class YardstickCLI():
             cmd_subparsers = subparser.add_subparsers(title='subcommands')
             self._find_actions(cmd_subparsers, command_object)
 
-        return parser
-
     def main(self, argv):
         '''run the command line interface'''
 
-        # get argument parser
-        parser = self._get_parser()
+        # register subcommands to parse additional command line arguments
+        parser = lambda subparsers: self._add_command_parsers(
+            YardstickCLI.categories, subparsers)
+        category_opt = cfg.SubCommandOpt("category",
+                                         title="Command categories",
+                                         help="Available categories",
+                                         handler=parser)
+        CONF.register_cli_opt(category_opt)
 
-        # parse command-line
-        args = parser.parse_args(argv)
+        # load CLI args and config files
+        CONF(argv, project="yardstick", version=self._version,
+             default_config_files=find_config_files(CONFIG_SEARCH_PATHS))
 
         # handle global opts
         logger = logging.getLogger('yardstick')
         logger.setLevel(logging.WARNING)
 
-        if args.verbose:
+        if CONF.verbose:
             logger.setLevel(logging.INFO)
 
-        if args.debug:
+        if CONF.debug:
             logger.setLevel(logging.DEBUG)
 
         # dispatch to category parser
-        args.func(args)
+        func = CONF.category.func
+        func(CONF.category)
diff --git a/yardstick/dispatcher/__init__.py b/yardstick/dispatcher/__init__.py
index 44278c1d2..b519efc7a 100644
--- a/yardstick/dispatcher/__init__.py
+++ b/yardstick/dispatcher/__init__.py
@@ -7,6 +7,16 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+from oslo_config import cfg
+
 import yardstick.common.utils as utils
 
 utils.import_modules_from_package("yardstick.dispatcher")
+
+CONF = cfg.CONF
+OPTS = [
+    cfg.StrOpt('dispatcher',
+               default='file',
+               help='Dispatcher to store data.'),
+]
+CONF.register_opts(OPTS)
diff --git a/yardstick/dispatcher/file.py b/yardstick/dispatcher/file.py
index dc39f152e..b6130005f 100644
--- a/yardstick/dispatcher/file.py
+++ b/yardstick/dispatcher/file.py
@@ -10,8 +10,25 @@
 import logging
 import json
 
+from oslo_config import cfg
+
 from yardstick.dispatcher.base import Base as DispatchBase
 
+CONF = cfg.CONF
+OPTS = [
+    cfg.StrOpt('file_path',
+               default='/tmp/yardstick.out',
+               help='Name and the location of the file to record '
+                    'data.'),
+    cfg.IntOpt('max_bytes',
+               default=0,
+               help='The max size of the file.'),
+    cfg.IntOpt('backup_count',
+               default=0,
+               help='The max number of the files to keep.'),
+]
+CONF.register_opts(OPTS, group="dispatcher_file")
+
 
 class FileDispatcher(DispatchBase):
     """Dispatcher class for recording data to a file.
@@ -19,27 +36,19 @@ class FileDispatcher(DispatchBase):
 
     __dispatcher_type__ = "File"
 
-    # TODO: make parameters below configurable, currently just hard coded
-    # Path of the file to record the data
-    file_path = "/tmp/yardstick.out"
-    # The max size of the file
-    max_bytes = 0
-    # The max number of the files to keep
-    backup_count = 0
-
     def __init__(self, conf):
         super(FileDispatcher, self).__init__(conf)
         self.log = None
 
         # if the directory and path are configured, then log to the file
-        if self.file_path:
+        if CONF.dispatcher_file.file_path:
             dispatcher_logger = logging.Logger('dispatcher.file')
             dispatcher_logger.setLevel(logging.INFO)
             # create rotating file handler which logs result
             rfh = logging.handlers.RotatingFileHandler(
-                self.conf.get('file_path', self.file_path),
-                maxBytes=self.max_bytes,
-                backupCount=self.backup_count,
+                self.conf.get('file_path', CONF.dispatcher_file.file_path),
+                maxBytes=CONF.dispatcher_file.max_bytes,
+                backupCount=CONF.dispatcher_file.backup_count,
                 encoding='utf8')
 
             rfh.setLevel(logging.INFO)
diff --git a/yardstick/dispatcher/http.py b/yardstick/dispatcher/http.py
index c3230adb2..703cfca62 100644
--- a/yardstick/dispatcher/http.py
+++ b/yardstick/dispatcher/http.py
@@ -9,13 +9,29 @@
 
 import json
 import logging
-
 import requests
 
+from oslo_config import cfg
+
 from yardstick.dispatcher.base import Base as DispatchBase
 
 LOG = logging.getLogger(__name__)
 
+CONF = cfg.CONF
+http_dispatcher_opts = [
+    cfg.StrOpt('target',
+               default='http://127.0.0.1:8000/results',
+               help='The target where the http request will be sent. '
+                    'If this is not set, no data will be posted. For '
+                    'example: target = http://hostname:1234/path'),
+    cfg.IntOpt('timeout',
+               default=5,
+               help='The max time in seconds to wait for a request to '
+                    'timeout.'),
+]
+
+CONF.register_opts(http_dispatcher_opts, group="dispatcher_http")
+
 
 class HttpDispatcher(DispatchBase):
     """Dispatcher class for posting data into a http target.
@@ -23,17 +39,11 @@ class HttpDispatcher(DispatchBase):
 
     __dispatcher_type__ = "Http"
 
-    # TODO: make parameters below configurable, currently just hard coded
-    # The target where the http request will be sent.
-    target = "http://127.0.0.1:8000/results"
-    # The max time in seconds to wait for a request to timeout.
-    timeout = 5
-
     def __init__(self, conf):
         super(HttpDispatcher, self).__init__(conf)
         self.headers = {'Content-type': 'application/json'}
-        self.timeout = self.timeout
-        self.target = self.target
+        self.timeout = CONF.dispatcher_http.timeout
+        self.target = CONF.dispatcher_http.target
 
     def record_result_data(self, data):
         if self.target == '':