Implement logfile collector and grep parser

JIRA: QTIP-207
JIRA: QTIP-208
Change-Id: Icc14d3097fb305e59df716636ef87504490c9d1b
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>

12 files changed, 123 insertions, 68 deletions

diff --git a/qtip/base/__init__.py b/qtip/base/__init__.py
index e69de29b..909703ed 100644
--- a/qtip/base/__init__.py
+++ b/qtip/base/__init__.py
@@ -0,0 +1,19 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseActor(object):
+    """abstract actor class"""
+
+    def __init__(self, config, parent=None):
+        self._config = config
+        self._parent = parent
+
+    def get_config(self, key, default=None):
+        return self._config.get(key, default)
diff --git a/qtip/base/constant.py b/qtip/base/constant.py
index 100ec009..09c635ac 100644
--- a/qtip/base/constant.py
+++ b/qtip/base/constant.py
@@ -35,6 +35,7 @@ class BaseProp(object):
     # content
     DESCRIPTION = 'description'
     WORKLOADS = 'workloads'
+    TYPE = 'type'
 
 
 class SpecProp(BaseProp):
diff --git a/qtip/cli/commands/cmd_metric.py b/qtip/cli/commands/cmd_metric.py
index d2fbd58f..aa4df1f4 100644
--- a/qtip/cli/commands/cmd_metric.py
+++ b/qtip/cli/commands/cmd_metric.py
@@ -27,7 +27,7 @@ def cmd_list(ctx):
     pass
 
 
-@cli.command('run', help='Run tests to collect Performance Metrics')
+@cli.command('run', help='Run tests to run Performance Metrics')
 @click.argument('name')
 @pass_context
 def cmd_run(ctx, name):
diff --git a/qtip/collector/__init__.py b/qtip/collector/__init__.py
index e69de29b..cc957ba4 100644
--- a/qtip/collector/__init__.py
+++ b/qtip/collector/__init__.py
@@ -0,0 +1,25 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from qtip.base.constant import BaseProp
+from qtip.collector.parser.grep import GrepParser
+
+
+class CollectorProp(BaseProp):
+    TYPE = 'type'
+    PARSERS = 'parsers'
+    PATHS = 'paths'
+
+
+def load_parser(type_name):
+    if type_name == GrepParser.TYPE:
+        return GrepParser
+    else:
+        raise Exception("Invalid parser type: {}".format(type_name))
diff --git a/qtip/collector/logfile.py b/qtip/collector/logfile.py
index 6ed5aafd..2c2e532f 100644
--- a/qtip/collector/logfile.py
+++ b/qtip/collector/logfile.py
@@ -7,36 +7,49 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from base import BaseCollector
+from itertools import chain
+from six.moves import reduce
+import os
 
-from qtip.collector.base import CollectorProp as CProp
+from qtip.base import BaseActor
+from qtip.collector import load_parser
+from qtip.collector import CollectorProp as CProp
 from qtip.loader.file import FileLoader
 
 
-class LogfileCollector(BaseCollector):
-    """collect performance metrics from log files"""
+class LogItem(BaseActor):
+    def find(self, filename, paths=None):
+        return self._parent.find(filename, paths)
 
+
+class LogfileCollector(BaseActor):
+    """run performance metrics from log files"""
     TYPE = 'logfile'
+    LOGS = 'logs'
+    PATHS = 'paths'
 
     def __init__(self, config, parent=None):
         super(LogfileCollector, self).__init__(config)
-        paths = [config[CProp.PATHS]] if CProp.PATHS in config else ['.']
-        self.loader = FileLoader('.', paths)
-        self._parent = parent
-
-    def collect(self):
-        captured = {}
-        for item in self._config[CProp.LOGS]:
-            captured.update(self._parse_log(item))
-        return captured
-
-    def _parse_log(self, log_item):
-        captured = {}
-        # TODO(yujunz) select parser by name
-        if CProp.GREP in log_item:
-            for rule in log_item[CProp.GREP]:
-                captured.update(self._grep(log_item[CProp.FILENAME], rule))
-        return captured
-
-    def _grep(self, filename, rule):
-        return {}
+        self._parent = parent  # plan
+        # TODO(yujunz) handle exception of invalid parent
+        dirname = os.path.dirname(self._parent.abspath)
+        paths = [os.path.join(dirname, p) for p in config.get(self.PATHS, [])]
+        self._loader = FileLoader('.', paths)
+
+    def run(self):
+        collected = []
+        for log_item_config in self._config[self.LOGS]:
+            log_item = LogItem(log_item_config, self)
+            matches = [load_parser(c[CProp.TYPE])(c, log_item).run()
+                       for c in log_item.get_config(CProp.PARSERS)]
+            collected = chain(collected, reduce(chain, matches))
+        return reduce(merge_matchobj_to_dict, collected, {'groups': (), 'groupdict': {}})
+
+    def find(self, filename, paths=None):
+        return self._loader.find(filename, paths)
+
+
+def merge_matchobj_to_dict(d, m):
+    d['groups'] = chain(d['groups'], m.groups())
+    d['groupdict'].update(m.groupdict())
+    return d
diff --git a/qtip/collector/parser/__init__.py b/qtip/collector/parser/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/collector/parser/__init__.py
diff --git a/qtip/collector/base.py b/qtip/collector/parser/grep.py
index e7f9756f..d7ada485 100644
--- a/qtip/collector/base.py
+++ b/qtip/collector/parser/grep.py
@@ -1,5 +1,5 @@
 ##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
+# Copyright (c) 2017 ZTE Corp and others.
 #
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
@@ -8,20 +8,26 @@
 ##############################################################################
 
 
-from qtip.base.constant import BaseProp
+import re
 
 
-class BaseCollector(object):
-    """performance metrics collector"""
-    def __init__(self, config):
-        self._config = config
+from qtip.base.constant import BaseProp
+from qtip.base import BaseActor
 
 
-class CollectorProp(BaseProp):
-    TYPE = 'type'
-    LOGS = 'logs'
+class GrepProp(BaseProp):
     FILENAME = 'filename'
-    GREP = 'grep'
     REGEX = 'regex'
-    CAPTURE = 'capture'
-    PATHS = 'path'
+
+
+class GrepParser(BaseActor):
+    TYPE = 'grep'
+
+    def run(self):
+        filename = self._parent.get_config(GrepProp.FILENAME)
+        return grep_in_file(self._parent.find(filename), self._config[GrepProp.REGEX])
+
+
+def grep_in_file(filename, regex):
+    with open(filename, 'r') as f:
+        return filter(lambda x: x is not None, [re.search(regex, line) for line in f])
diff --git a/qtip/loader/file.py b/qtip/loader/file.py
index 00f94818..0ea4d5b6 100644
--- a/qtip/loader/file.py
+++ b/qtip/loader/file.py
@@ -25,12 +25,12 @@ class FileLoader(BaseLoader):
     _paths = [ROOT_DIR]
 
     def __init__(self, name, paths=None):
-        self._file = name
-        self._abspath = self._find(name, paths=paths)
+        self._filename = name
+        self.abspath = self.find(name, paths=paths)
 
-    def _find(self, name, paths=None):
+    def find(self, name, paths=None):
         """find a specification in searching paths"""
-        paths = self._paths if paths is None else paths
+        paths = [self.abspath] if paths is None else paths
         for p in paths:
             abspath = path.join(p, self.RELATIVE_PATH, name)
             if path.exists(abspath):
@@ -47,4 +47,4 @@ class FileLoader(BaseLoader):
             item = cls(name, paths=paths)
             yield {
                 BaseProp.NAME: name,
-                BaseProp.ABSPATH: item._abspath}
+                BaseProp.ABSPATH: item.abspath}
diff --git a/qtip/loader/module.py b/qtip/loader/module.py
deleted file mode 100644
index 05cb1b76..00000000
--- a/qtip/loader/module.py
+++ /dev/null
@@ -1,19 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from qtip.collector.logfile import LogfileCollector
-
-
-# TODO(yujunz) more elegant way to load module dynamically
-def load_collector(type_name):
-    if type_name == LogfileCollector.TYPE:
-        return LogfileCollector
-    else:
-        raise Exception("Invalid collector type: {}".format(type_name))
diff --git a/qtip/loader/plan.py b/qtip/loader/plan.py
index 0fd9ff52..e15651a3 100644
--- a/qtip/loader/plan.py
+++ b/qtip/loader/plan.py
@@ -9,12 +9,20 @@
 
 
 from qtip.base.constant import BaseProp
-from qtip.collector.base import CollectorProp as CProp
-from qtip.loader.module import load_collector
+from qtip.collector import CollectorProp as CProp
+from qtip.collector.logfile import LogfileCollector
 from qtip.loader.yaml_file import YamlFileLoader
 from qtip.loader.qpi import QPISpec
 
 
+# TODO(yujunz) more elegant way to load module dynamically
+def load_collector(type_name):
+    if type_name == LogfileCollector.TYPE:
+        return LogfileCollector
+    else:
+        raise Exception("Invalid collector type: {}".format(type_name))
+
+
 class Plan(YamlFileLoader):
     """
     a benchmark plan is consist of configuration and a QPI list
diff --git a/qtip/loader/yaml_file.py b/qtip/loader/yaml_file.py
index ccaee8db..8b78a47c 100644
--- a/qtip/loader/yaml_file.py
+++ b/qtip/loader/yaml_file.py
@@ -20,9 +20,11 @@ class YamlFileLoader(FileLoader):
 
     def __init__(self, name, paths=None):
         super(YamlFileLoader, self).__init__(name, paths)
-        with open(self._abspath, 'r') as stream:
+        abspath = self.abspath
+
+        with open(abspath, 'r') as stream:
             content = yaml.safe_load(stream)
             if not isinstance(content, dict):
-                raise InvalidContent(self._abspath)
+                raise InvalidContent(abspath)
             self.content = content
             self.name = content.get(BaseProp.NAME, path.splitext(name)[0])
diff --git a/qtip/runner/__init__.py b/qtip/runner/__init__.py
index 1db8498e..79c38850 100644
--- a/qtip/runner/__init__.py
+++ b/qtip/runner/__init__.py
@@ -28,16 +28,16 @@ class Runner(object):
         if driver_name == 'random':
             self.driver = RandomDriver()
         else:
-            raise NotFound(driver_name, package=PkgName.DRIVER)
+            raise NotFound(driver_name, heystack=PkgName.DRIVER)
 
         if collector_name == 'stdout':
             self.collector = StdoutCollector()
         else:
             raise NotFound(collector_name,
-                           package=PkgName.COLLECTOR)
+                           heystack=PkgName.COLLECTOR)
 
         if reporter_name == 'console':
             self.reporter = ConsoleReporter()
         else:
             raise NotFound(reporter_name,
-                           package=PkgName.REPORTER)
+                           heystack=PkgName.REPORTER)