Implement logfile collector and grep parser

JIRA: QTIP-207
JIRA: QTIP-208
Change-Id: Icc14d3097fb305e59df716636ef87504490c9d1b
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>

4 files changed, 80 insertions, 36 deletions

diff --git a/qtip/collector/__init__.py b/qtip/collector/__init__.py
index e69de29b..cc957ba4 100644
--- a/qtip/collector/__init__.py
+++ b/qtip/collector/__init__.py
@@ -0,0 +1,25 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from qtip.base.constant import BaseProp
+from qtip.collector.parser.grep import GrepParser
+
+
+class CollectorProp(BaseProp):
+    TYPE = 'type'
+    PARSERS = 'parsers'
+    PATHS = 'paths'
+
+
+def load_parser(type_name):
+    if type_name == GrepParser.TYPE:
+        return GrepParser
+    else:
+        raise Exception("Invalid parser type: {}".format(type_name))
diff --git a/qtip/collector/logfile.py b/qtip/collector/logfile.py
index 6ed5aafd..2c2e532f 100644
--- a/qtip/collector/logfile.py
+++ b/qtip/collector/logfile.py
@@ -7,36 +7,49 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from base import BaseCollector
+from itertools import chain
+from six.moves import reduce
+import os
 
-from qtip.collector.base import CollectorProp as CProp
+from qtip.base import BaseActor
+from qtip.collector import load_parser
+from qtip.collector import CollectorProp as CProp
 from qtip.loader.file import FileLoader
 
 
-class LogfileCollector(BaseCollector):
-    """collect performance metrics from log files"""
+class LogItem(BaseActor):
+    def find(self, filename, paths=None):
+        return self._parent.find(filename, paths)
 
+
+class LogfileCollector(BaseActor):
+    """run performance metrics from log files"""
     TYPE = 'logfile'
+    LOGS = 'logs'
+    PATHS = 'paths'
 
     def __init__(self, config, parent=None):
         super(LogfileCollector, self).__init__(config)
-        paths = [config[CProp.PATHS]] if CProp.PATHS in config else ['.']
-        self.loader = FileLoader('.', paths)
-        self._parent = parent
-
-    def collect(self):
-        captured = {}
-        for item in self._config[CProp.LOGS]:
-            captured.update(self._parse_log(item))
-        return captured
-
-    def _parse_log(self, log_item):
-        captured = {}
-        # TODO(yujunz) select parser by name
-        if CProp.GREP in log_item:
-            for rule in log_item[CProp.GREP]:
-                captured.update(self._grep(log_item[CProp.FILENAME], rule))
-        return captured
-
-    def _grep(self, filename, rule):
-        return {}
+        self._parent = parent  # plan
+        # TODO(yujunz) handle exception of invalid parent
+        dirname = os.path.dirname(self._parent.abspath)
+        paths = [os.path.join(dirname, p) for p in config.get(self.PATHS, [])]
+        self._loader = FileLoader('.', paths)
+
+    def run(self):
+        collected = []
+        for log_item_config in self._config[self.LOGS]:
+            log_item = LogItem(log_item_config, self)
+            matches = [load_parser(c[CProp.TYPE])(c, log_item).run()
+                       for c in log_item.get_config(CProp.PARSERS)]
+            collected = chain(collected, reduce(chain, matches))
+        return reduce(merge_matchobj_to_dict, collected, {'groups': (), 'groupdict': {}})
+
+    def find(self, filename, paths=None):
+        return self._loader.find(filename, paths)
+
+
+def merge_matchobj_to_dict(d, m):
+    d['groups'] = chain(d['groups'], m.groups())
+    d['groupdict'].update(m.groupdict())
+    return d
diff --git a/qtip/collector/parser/__init__.py b/qtip/collector/parser/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/collector/parser/__init__.py
diff --git a/qtip/collector/base.py b/qtip/collector/parser/grep.py
index e7f9756f..d7ada485 100644
--- a/qtip/collector/base.py
+++ b/qtip/collector/parser/grep.py
@@ -1,5 +1,5 @@
 ##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
+# Copyright (c) 2017 ZTE Corp and others.
 #
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
@@ -8,20 +8,26 @@
 ##############################################################################
 
 
-from qtip.base.constant import BaseProp
+import re
 
 
-class BaseCollector(object):
-    """performance metrics collector"""
-    def __init__(self, config):
-        self._config = config
+from qtip.base.constant import BaseProp
+from qtip.base import BaseActor
 
 
-class CollectorProp(BaseProp):
-    TYPE = 'type'
-    LOGS = 'logs'
+class GrepProp(BaseProp):
     FILENAME = 'filename'
-    GREP = 'grep'
     REGEX = 'regex'
-    CAPTURE = 'capture'
-    PATHS = 'path'
+
+
+class GrepParser(BaseActor):
+    TYPE = 'grep'
+
+    def run(self):
+        filename = self._parent.get_config(GrepProp.FILENAME)
+        return grep_in_file(self._parent.find(filename), self._config[GrepProp.REGEX])
+
+
+def grep_in_file(filename, regex):
+    with open(filename, 'r') as f:
+        return filter(lambda x: x is not None, [re.search(regex, line) for line in f])