From cdb8e1be12b724e9f68c7e46c9a653ec37c502cb Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Sun, 5 Feb 2017 12:24:30 +0800 Subject: Implement logfile collector and grep parser JIRA: QTIP-207 JIRA: QTIP-208 Change-Id: Icc14d3097fb305e59df716636ef87504490c9d1b Signed-off-by: Yujun Zhang --- qtip/collector/__init__.py | 25 ++++++++++++++++ qtip/collector/base.py | 27 ----------------- qtip/collector/logfile.py | 61 ++++++++++++++++++++++++--------------- qtip/collector/parser/__init__.py | 0 qtip/collector/parser/grep.py | 33 +++++++++++++++++++++ 5 files changed, 95 insertions(+), 51 deletions(-) delete mode 100644 qtip/collector/base.py create mode 100644 qtip/collector/parser/__init__.py create mode 100644 qtip/collector/parser/grep.py (limited to 'qtip/collector') diff --git a/qtip/collector/__init__.py b/qtip/collector/__init__.py index e69de29b..cc957ba4 100644 --- a/qtip/collector/__init__.py +++ b/qtip/collector/__init__.py @@ -0,0 +1,25 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from qtip.base.constant import BaseProp +from qtip.collector.parser.grep import GrepParser + + +class CollectorProp(BaseProp): + TYPE = 'type' + PARSERS = 'parsers' + PATHS = 'paths' + + +def load_parser(type_name): + if type_name == GrepParser.TYPE: + return GrepParser + else: + raise Exception("Invalid parser type: {}".format(type_name)) diff --git a/qtip/collector/base.py b/qtip/collector/base.py deleted file mode 100644 index e7f9756f..00000000 --- a/qtip/collector/base.py +++ /dev/null @@ -1,27 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from qtip.base.constant import BaseProp - - -class BaseCollector(object): - """performance metrics collector""" - def __init__(self, config): - self._config = config - - -class CollectorProp(BaseProp): - TYPE = 'type' - LOGS = 'logs' - FILENAME = 'filename' - GREP = 'grep' - REGEX = 'regex' - CAPTURE = 'capture' - PATHS = 'path' diff --git a/qtip/collector/logfile.py b/qtip/collector/logfile.py index 6ed5aafd..2c2e532f 100644 --- a/qtip/collector/logfile.py +++ b/qtip/collector/logfile.py @@ -7,36 +7,49 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from base import BaseCollector +from itertools import chain +from six.moves import reduce +import os -from qtip.collector.base import CollectorProp as CProp +from qtip.base import BaseActor +from qtip.collector import load_parser +from qtip.collector import CollectorProp as CProp from qtip.loader.file import FileLoader -class LogfileCollector(BaseCollector): - """collect performance metrics from log files""" +class LogItem(BaseActor): + def find(self, filename, paths=None): + return self._parent.find(filename, paths) + +class LogfileCollector(BaseActor): + """run performance metrics from log files""" TYPE = 'logfile' + LOGS = 'logs' + PATHS = 'paths' def __init__(self, config, parent=None): super(LogfileCollector, self).__init__(config) - paths = [config[CProp.PATHS]] if CProp.PATHS in config else ['.'] - self.loader = FileLoader('.', paths) - self._parent = parent - - def collect(self): - captured = {} - for item in self._config[CProp.LOGS]: - captured.update(self._parse_log(item)) - return captured - - def _parse_log(self, log_item): - captured = {} - # TODO(yujunz) select parser by name - if CProp.GREP in log_item: - for rule in log_item[CProp.GREP]: - captured.update(self._grep(log_item[CProp.FILENAME], rule)) - return captured - - def _grep(self, filename, rule): - return {} + self._parent = parent # plan + # TODO(yujunz) handle exception of invalid parent + dirname = os.path.dirname(self._parent.abspath) + paths = [os.path.join(dirname, p) for p in config.get(self.PATHS, [])] + self._loader = FileLoader('.', paths) + + def run(self): + collected = [] + for log_item_config in self._config[self.LOGS]: + log_item = LogItem(log_item_config, self) + matches = [load_parser(c[CProp.TYPE])(c, log_item).run() + for c in log_item.get_config(CProp.PARSERS)] + collected = chain(collected, reduce(chain, matches)) + return reduce(merge_matchobj_to_dict, collected, {'groups': (), 'groupdict': {}}) + + def find(self, filename, paths=None): + return self._loader.find(filename, paths) + + +def merge_matchobj_to_dict(d, m): + d['groups'] = chain(d['groups'], m.groups()) + d['groupdict'].update(m.groupdict()) + return d diff --git a/qtip/collector/parser/__init__.py b/qtip/collector/parser/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qtip/collector/parser/grep.py b/qtip/collector/parser/grep.py new file mode 100644 index 00000000..d7ada485 --- /dev/null +++ b/qtip/collector/parser/grep.py @@ -0,0 +1,33 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +import re + + +from qtip.base.constant import BaseProp +from qtip.base import BaseActor + + +class GrepProp(BaseProp): + FILENAME = 'filename' + REGEX = 'regex' + + +class GrepParser(BaseActor): + TYPE = 'grep' + + def run(self): + filename = self._parent.get_config(GrepProp.FILENAME) + return grep_in_file(self._parent.find(filename), self._config[GrepProp.REGEX]) + + +def grep_in_file(filename, regex): + with open(filename, 'r') as f: + return filter(lambda x: x is not None, [re.search(regex, line) for line in f]) -- cgit 1.2.3-korg