From f2d021c72b38845954755bab54aa13b4b2aad725 Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Thu, 19 Jan 2017 16:58:46 +0800 Subject: Refactor loader classes - rename BaseLoader to YamlFileLoader as base class of QTIP specs loader - create an abstract BaseLoader - create FileLoader for logfile collector Change-Id: I0c992cd847fc0dce4fdd73a13c1cdbc406c84532 Signed-off-by: Yujun Zhang --- qtip/base/constant.py | 4 +-- qtip/collector/base.py | 2 ++ qtip/collector/logfile.py | 24 +++++++++++++++ qtip/loader/base.py | 53 +--------------------------------- qtip/loader/file.py | 50 ++++++++++++++++++++++++++++++++ qtip/loader/metric.py | 4 +-- qtip/loader/plan.py | 9 ++++-- qtip/loader/qpi.py | 4 +-- qtip/loader/yaml_file.py | 33 +++++++++++++++++++++ tests/data/benchmarks/plan/doctor.yaml | 37 ++++++++++++++++++++++++ tests/data/opt/plan/doctor.yaml | 37 ------------------------ tests/unit/loader/plan_test.py | 2 +- 12 files changed, 161 insertions(+), 98 deletions(-) create mode 100644 qtip/loader/file.py create mode 100644 qtip/loader/yaml_file.py create mode 100644 tests/data/benchmarks/plan/doctor.yaml delete mode 100644 tests/data/opt/plan/doctor.yaml diff --git a/qtip/base/constant.py b/qtip/base/constant.py index 58cc0cc1..ddd07e9d 100644 --- a/qtip/base/constant.py +++ b/qtip/base/constant.py @@ -64,8 +64,8 @@ class PlanProp(BaseProp): class CollectorProp(BaseProp): LOGS = 'logs' FILENAME = 'filename' - PATTERNS = 'patterns' - MATCH = 'match' + GREP = 'grep' + REGEX = 'regex' CAPTURE = 'capture' diff --git a/qtip/collector/base.py b/qtip/collector/base.py index cd8fc797..2a25455c 100644 --- a/qtip/collector/base.py +++ b/qtip/collector/base.py @@ -10,3 +10,5 @@ class BaseCollector(object): """performance metrics collector""" + def __init__(self, config): + self._config = config diff --git a/qtip/collector/logfile.py b/qtip/collector/logfile.py index 6528ea9f..19780aaa 100644 --- a/qtip/collector/logfile.py +++ b/qtip/collector/logfile.py @@ -9,6 +9,30 @@ from base import BaseCollector +from qtip.base.constant import CollectorProp as CProp +from qtip.loader.file import FileLoader + class LogfileCollector(BaseCollector): """collect performance metrics from log files""" + + def __init__(self, config, paths=None): + super(LogfileCollector, self).__init__(config) + self.loader = FileLoader('.', paths) + + def collect(self): + captured = {} + for item in self._config[CProp.LOGS]: + captured.update(self._parse_log(item)) + return captured + + def _parse_log(self, log_item): + captured = {} + # TODO(yujunz) select parser by name + if CProp.GREP in log_item: + for rule in log_item[CProp.GREP]: + captured.update(self._grep(log_item[CProp.FILENAME], rule)) + return captured + + def _grep(self, filename, rule): + return {} diff --git a/qtip/loader/base.py b/qtip/loader/base.py index a0e5d031..abc254bf 100644 --- a/qtip/loader/base.py +++ b/qtip/loader/base.py @@ -7,57 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from collections import defaultdict -from itertools import chain -from os import listdir -from os import path -import yaml - -from qtip.base.error import InvalidFormat, NotFound -from qtip.base.constant import BaseProp - - -ROOT_DIR = path.join(path.dirname(__file__), path.pardir, path.pardir, - 'benchmarks') - class BaseLoader(object): - """Abstract class of QTIP benchmark loader""" - RELATIVE_PATH = '.' - _paths = [ROOT_DIR] - - def __init__(self, name, paths=None): - self._file = name - self._abspath = self._find(name, paths=paths) - content = defaultdict(lambda: None) - - try: - content.update(yaml.safe_load(file(self._abspath))) - except yaml.YAMLError: - # TODO(yujunz) log yaml error - raise InvalidFormat(self._abspath) - - self.name = content[BaseProp.NAME] or path.splitext(name)[0] - self.content = content - - def _find(self, name, paths=None): - """find a benchmark in searching paths""" - paths = self._paths if paths is None else paths - for p in paths: - abspath = path.join(p, self.RELATIVE_PATH, name) - if path.exists(abspath): - return abspath - raise NotFound(name, paths) - - @classmethod - def list_all(cls, paths=None): - """list all available benchmarks""" - paths = cls._paths if paths is None else paths - names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH)) - for p in paths]) - for name in names: - item = cls(name, paths=paths) - yield { - BaseProp.NAME: name, - BaseProp.ABSPATH: item._abspath, - BaseProp.CONTENT: item.content} + """Abstract class of QTIP loader""" diff --git a/qtip/loader/file.py b/qtip/loader/file.py new file mode 100644 index 00000000..00f94818 --- /dev/null +++ b/qtip/loader/file.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from itertools import chain +from os import listdir +from os import path + +from qtip.base.constant import BaseProp +from qtip.base.error import NotFound +from qtip.loader.base import BaseLoader + + +ROOT_DIR = path.join(path.dirname(__file__), path.pardir, path.pardir, + 'benchmarks') + + +class FileLoader(BaseLoader): + RELATIVE_PATH = '.' + _paths = [ROOT_DIR] + + def __init__(self, name, paths=None): + self._file = name + self._abspath = self._find(name, paths=paths) + + def _find(self, name, paths=None): + """find a specification in searching paths""" + paths = self._paths if paths is None else paths + for p in paths: + abspath = path.join(p, self.RELATIVE_PATH, name) + if path.exists(abspath): + return abspath + raise NotFound(name, paths) + + @classmethod + def list_all(cls, paths=None): + """list all available specification""" + paths = cls._paths if paths is None else paths + names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH)) + for p in paths]) + for name in names: + item = cls(name, paths=paths) + yield { + BaseProp.NAME: name, + BaseProp.ABSPATH: item._abspath} diff --git a/qtip/loader/metric.py b/qtip/loader/metric.py index 8b6fa5d3..842fcdbf 100644 --- a/qtip/loader/metric.py +++ b/qtip/loader/metric.py @@ -7,10 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from base import BaseLoader +from yaml_file import YamlFileLoader -class MetricSpec(BaseLoader): +class MetricSpec(YamlFileLoader): """metrics in QTIP are categorized by performance test tools, such as dhrystone, whetstone and etc""" RELATIVE_PATH = 'metric' diff --git a/qtip/loader/plan.py b/qtip/loader/plan.py index c1ee00be..6f1764e2 100644 --- a/qtip/loader/plan.py +++ b/qtip/loader/plan.py @@ -9,11 +9,12 @@ from qtip.base.constant import PlanProp -from qtip.loader.base import BaseLoader +from qtip.collector.logfile import LogfileCollector +from qtip.loader.yaml_file import YamlFileLoader from qtip.loader.qpi import QPISpec -class Plan(BaseLoader): +class Plan(YamlFileLoader): """ a benchmark plan is consist of configuration and a QPI list """ @@ -26,3 +27,7 @@ class Plan(BaseLoader): self.qpis = [QPISpec(qpi, paths=paths) for qpi in self.content[PlanProp.QPIS]] self.info = self.content[PlanProp.INFO] + _config = self.content[PlanProp.CONFIG] + + # TODO(yujunz) create collector by name + self.collector = LogfileCollector(_config[PlanProp.COLLECTOR], paths) diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py index ef6e065e..73da61e9 100644 --- a/qtip/loader/qpi.py +++ b/qtip/loader/qpi.py @@ -7,14 +7,14 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from base import BaseLoader +from yaml_file import YamlFileLoader from metric import MetricSpec from qtip.base.constant import SpecProp from qtip.util.formula import Formula -class QPISpec(BaseLoader): +class QPISpec(YamlFileLoader): """ a QPI specification defines how to calculate a performance index from collected metrics. diff --git a/qtip/loader/yaml_file.py b/qtip/loader/yaml_file.py new file mode 100644 index 00000000..f1cd4614 --- /dev/null +++ b/qtip/loader/yaml_file.py @@ -0,0 +1,33 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from collections import defaultdict +from os import path +import yaml + +from qtip.base.error import InvalidFormat +from qtip.base.constant import BaseProp +from qtip.loader.file import FileLoader + + +class YamlFileLoader(FileLoader): + """load content from yaml file""" + + def __init__(self, name, paths=None): + super(YamlFileLoader, self).__init__(name, paths) + content = defaultdict(lambda: None) + + try: + content.update(yaml.safe_load(file(self._abspath))) + except yaml.YAMLError: + # TODO(yujunz) log yaml error + raise InvalidFormat(self._abspath) + + self.name = content[BaseProp.NAME] or path.splitext(name)[0] + self.content = content diff --git a/tests/data/benchmarks/plan/doctor.yaml b/tests/data/benchmarks/plan/doctor.yaml new file mode 100644 index 00000000..6c95077b --- /dev/null +++ b/tests/data/benchmarks/plan/doctor.yaml @@ -0,0 +1,37 @@ +name: doctor performance profiling +description: create performance profile for OPNFV doctor notification time +info: + facility: local + engineer: local +config: + driver: sample + collector: + - name: logfile + logs: + - filename: doctor_consumer.log + # 2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63 + grep: + - regex: 'doctor consumer notified at \d+(\.\d+)?$' + capture: notified consumer + - filename: doctor_inspector.log + # 2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3 + # 2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm() error at 1482894965.3 + # 2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51 + grep: + - regex: 'event posted at \d+(\.\d+)?$' + capture: posted event + - regex: 'doctor mark vm\(.*\) error at \d+(\.\d+)?$' + capture: marked VM error + - regex: 'doctor mark host\(.*\) down at \d+(\.\d+)?$' + capture: marked host down + - filename: disable_network.log + # doctor set host down at 1482894965.164096803 + grep: + - regex: 'doctor set host down at \d+(\.\d+)?$' + capture: set host down + reporter: + name: console + # transform collected data into timeline + transformer: timeline +QPIs: + - fake-qpi.yaml diff --git a/tests/data/opt/plan/doctor.yaml b/tests/data/opt/plan/doctor.yaml deleted file mode 100644 index 48b4c955..00000000 --- a/tests/data/opt/plan/doctor.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: doctor performance profiling -description: create performance profile for OPNFV doctor notification time -info: - facility: local - engineer: local -config: - driver: sample - collector: - - name: logfile - logs: - - filename: doctor_consumer.log - # 2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63 - patterns: - - match: 'doctor consumer notified at \d+(\.\d+)?$' - capture: notified consumer - - filename: doctor_inspector.log - # 2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3 - # 2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm() error at 1482894965.3 - # 2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51 - patterns: - - match: 'event posted at \d+(\.\d+)?$' - capture: posted event - - match: 'doctor mark vm\(.*\) error at \d+(\.\d+)?$' - capture: marked VM error - - match: 'doctor mark host\(.*\) down at \d+(\.\d+)?$' - capture: marked host down - - filename: disable_network.log - # doctor set host down at 1482894965.164096803 - patterns: - - match: 'doctor set host down at \d+(\.\d+)?$' - capture: set host down - reporter: - name: console - # transform collected data into timeline - transformer: timeline -QPIs: - - fake-qpi.yaml diff --git a/tests/unit/loader/plan_test.py b/tests/unit/loader/plan_test.py index b57bcfb5..32837f8f 100644 --- a/tests/unit/loader/plan_test.py +++ b/tests/unit/loader/plan_test.py @@ -27,7 +27,7 @@ def test_init(plan): def test_list_all(benchmarks_root): plan_list = Plan.list_all(paths=[benchmarks_root]) - assert len(list(plan_list)) is 1 + assert len(list(plan_list)) is 2 for desc in plan_list: assert PlanProp.NAME in desc assert PlanProp.CONTENT in desc -- cgit 1.2.3-korg