aboutsummaryrefslogtreecommitdiffstats
path: root/qtip/collector
diff options
context:
space:
mode:
Diffstat (limited to 'qtip/collector')
-rw-r--r--qtip/collector/__init__.py25
-rw-r--r--qtip/collector/logfile.py61
-rw-r--r--qtip/collector/parser/__init__.py0
-rw-r--r--qtip/collector/parser/grep.py (renamed from qtip/collector/base.py)30
4 files changed, 80 insertions, 36 deletions
diff --git a/qtip/collector/__init__.py b/qtip/collector/__init__.py
index e69de29b..cc957ba4 100644
--- a/qtip/collector/__init__.py
+++ b/qtip/collector/__init__.py
@@ -0,0 +1,25 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from qtip.base.constant import BaseProp
+from qtip.collector.parser.grep import GrepParser
+
+
+class CollectorProp(BaseProp):
+ TYPE = 'type'
+ PARSERS = 'parsers'
+ PATHS = 'paths'
+
+
+def load_parser(type_name):
+ if type_name == GrepParser.TYPE:
+ return GrepParser
+ else:
+ raise Exception("Invalid parser type: {}".format(type_name))
diff --git a/qtip/collector/logfile.py b/qtip/collector/logfile.py
index 6ed5aafd..2c2e532f 100644
--- a/qtip/collector/logfile.py
+++ b/qtip/collector/logfile.py
@@ -7,36 +7,49 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from base import BaseCollector
+from itertools import chain
+from six.moves import reduce
+import os
-from qtip.collector.base import CollectorProp as CProp
+from qtip.base import BaseActor
+from qtip.collector import load_parser
+from qtip.collector import CollectorProp as CProp
from qtip.loader.file import FileLoader
-class LogfileCollector(BaseCollector):
- """collect performance metrics from log files"""
+class LogItem(BaseActor):
+ def find(self, filename, paths=None):
+ return self._parent.find(filename, paths)
+
+class LogfileCollector(BaseActor):
+ """run performance metrics from log files"""
TYPE = 'logfile'
+ LOGS = 'logs'
+ PATHS = 'paths'
def __init__(self, config, parent=None):
super(LogfileCollector, self).__init__(config)
- paths = [config[CProp.PATHS]] if CProp.PATHS in config else ['.']
- self.loader = FileLoader('.', paths)
- self._parent = parent
-
- def collect(self):
- captured = {}
- for item in self._config[CProp.LOGS]:
- captured.update(self._parse_log(item))
- return captured
-
- def _parse_log(self, log_item):
- captured = {}
- # TODO(yujunz) select parser by name
- if CProp.GREP in log_item:
- for rule in log_item[CProp.GREP]:
- captured.update(self._grep(log_item[CProp.FILENAME], rule))
- return captured
-
- def _grep(self, filename, rule):
- return {}
+ self._parent = parent # plan
+ # TODO(yujunz) handle exception of invalid parent
+ dirname = os.path.dirname(self._parent.abspath)
+ paths = [os.path.join(dirname, p) for p in config.get(self.PATHS, [])]
+ self._loader = FileLoader('.', paths)
+
+ def run(self):
+ collected = []
+ for log_item_config in self._config[self.LOGS]:
+ log_item = LogItem(log_item_config, self)
+ matches = [load_parser(c[CProp.TYPE])(c, log_item).run()
+ for c in log_item.get_config(CProp.PARSERS)]
+ collected = chain(collected, reduce(chain, matches))
+ return reduce(merge_matchobj_to_dict, collected, {'groups': (), 'groupdict': {}})
+
+ def find(self, filename, paths=None):
+ return self._loader.find(filename, paths)
+
+
+def merge_matchobj_to_dict(d, m):
+ d['groups'] = chain(d['groups'], m.groups())
+ d['groupdict'].update(m.groupdict())
+ return d
diff --git a/qtip/collector/parser/__init__.py b/qtip/collector/parser/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/collector/parser/__init__.py
diff --git a/qtip/collector/base.py b/qtip/collector/parser/grep.py
index e7f9756f..d7ada485 100644
--- a/qtip/collector/base.py
+++ b/qtip/collector/parser/grep.py
@@ -1,5 +1,5 @@
##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
+# Copyright (c) 2017 ZTE Corp and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
@@ -8,20 +8,26 @@
##############################################################################
-from qtip.base.constant import BaseProp
+import re
-class BaseCollector(object):
- """performance metrics collector"""
- def __init__(self, config):
- self._config = config
+from qtip.base.constant import BaseProp
+from qtip.base import BaseActor
-class CollectorProp(BaseProp):
- TYPE = 'type'
- LOGS = 'logs'
+class GrepProp(BaseProp):
FILENAME = 'filename'
- GREP = 'grep'
REGEX = 'regex'
- CAPTURE = 'capture'
- PATHS = 'path'
+
+
+class GrepParser(BaseActor):
+ TYPE = 'grep'
+
+ def run(self):
+ filename = self._parent.get_config(GrepProp.FILENAME)
+ return grep_in_file(self._parent.find(filename), self._config[GrepProp.REGEX])
+
+
+def grep_in_file(filename, regex):
+ with open(filename, 'r') as f:
+ return filter(lambda x: x is not None, [re.search(regex, line) for line in f])