diff options
author | Yujun Zhang <zhang.yujunz@zte.com.cn> | 2016-12-27 16:57:28 +0800 |
---|---|---|
committer | Yujun Zhang <zhang.yujunz@zte.com.cn> | 2017-01-03 17:34:11 +0800 |
commit | decad5f7fdb212c4f78fc31f00e591eaba104450 (patch) | |
tree | 93ee0cd48f991c83bd9e13a4814cbced2377ca21 /qtip | |
parent | 0dbc16c41646cbec79b65ec2ac60b8965cd22db9 (diff) |
Add module skeletons
- Remove `suite` and `case` which seems redundant at the moment
- Rename `qtip.spec` to `qtip.loader` since the class loads spec
- Add driver `sample` which will generate random performance data
- Add collector `logfile` which will collect result from log files
- Add reporter `console` which will print report to console
- Add exception classes
See updated class diagram in https://wiki.opnfv.org/display/qtip/Design
JIRA: QTIP-148
JIRA: QTIP-193
Change-Id: Idf751d33428176a26224e4443100a142feda6f32
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>
Diffstat (limited to 'qtip')
-rw-r--r-- | qtip/base/constant.py | 50 | ||||
-rw-r--r-- | qtip/base/error.py | 23 | ||||
-rw-r--r-- | qtip/collector/__init__.py (renamed from qtip/drivers/__init__.py) | 0 | ||||
-rw-r--r-- | qtip/collector/base.py (renamed from qtip/drivers/base.py) | 4 | ||||
-rw-r--r-- | qtip/collector/logfile.py (renamed from qtip/agent/reporter.py) | 7 | ||||
-rw-r--r-- | qtip/driver/__init__.py (renamed from qtip/spec/__init__.py) | 0 | ||||
-rw-r--r-- | qtip/driver/ansible.py (renamed from qtip/drivers/ansible.py) | 0 | ||||
-rw-r--r-- | qtip/driver/base.py (renamed from qtip/runner/case.py) | 15 | ||||
-rw-r--r-- | qtip/driver/sample.py | 15 | ||||
-rw-r--r-- | qtip/driver/yardstick.py (renamed from qtip/drivers/yardstick.py) | 0 | ||||
-rw-r--r-- | qtip/loader/__init__.py | 0 | ||||
-rw-r--r-- | qtip/loader/base.py (renamed from qtip/base/benchmark.py) | 69 | ||||
-rw-r--r-- | qtip/loader/metric.py (renamed from qtip/spec/metric.py) | 6 | ||||
-rw-r--r-- | qtip/loader/qpi.py (renamed from qtip/spec/qpi.py) | 4 | ||||
-rw-r--r-- | qtip/reporter/__init__.py | 0 | ||||
-rw-r--r-- | qtip/reporter/base.py | 14 | ||||
-rw-r--r-- | qtip/reporter/console.py (renamed from qtip/agent/collector.py) | 7 | ||||
-rw-r--r-- | qtip/runner/base.py | 45 | ||||
-rw-r--r-- | qtip/runner/plan.py | 21 | ||||
-rw-r--r-- | qtip/runner/suite.py | 22 |
20 files changed, 199 insertions, 103 deletions
diff --git a/qtip/base/constant.py b/qtip/base/constant.py new file mode 100644 index 00000000..187f0706 --- /dev/null +++ b/qtip/base/constant.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class AlgoName(object): + """algorithm names""" + ARITHMETIC_MEAN = 'arithmetic mean' + WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean' + GEOMETRIC_MEAN = 'geometric mean' + WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean' + + +class PkgName(object): + """QTIP package names""" + COLLECTOR = 'collector' + DRIVER = 'driver' + REPORTER = 'reporter' + RUNNER = 'runner' + SPEC = 'spec' + + +class PropName(object): + """property names""" + # list + NAME = 'name' + CONTENT = 'content' + ABSPATH = 'abspath' + # content + name = 'name' + DESCRIPTION = 'description' + # spec + SECTIONS = 'sections' + WEIGHT = 'weight' + ALGORITHM = 'algorithm' + METRICS = 'metrics' + WORKLOADS = 'workloads' + # plan + CONFIG = 'config' + FACILITY = 'facility' + ENGINEER = 'engineer' + DRIVER = 'driver' + COLLECTOR = 'collector' + REPORTER = 'reporter' + QPIS = 'QPIs' diff --git a/qtip/base/error.py b/qtip/base/error.py new file mode 100644 index 00000000..d364c532 --- /dev/null +++ b/qtip/base/error.py @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class QtipError(Exception): + pass + + +class InvalidFormat(QtipError): + def __init__(self, filename): + self.filename = filename + + +class NotFound(QtipError): + def __init__(self, module, package='qtip'): + self.package = package + self.module = module diff --git a/qtip/drivers/__init__.py b/qtip/collector/__init__.py index e69de29b..e69de29b 100644 --- a/qtip/drivers/__init__.py +++ b/qtip/collector/__init__.py diff --git a/qtip/drivers/base.py b/qtip/collector/base.py index 1aa8d8ad..cd8fc797 100644 --- a/qtip/drivers/base.py +++ b/qtip/collector/base.py @@ -8,5 +8,5 @@ ############################################################################## -class BaseDriver(object): - """performance testing tool driver""" +class BaseCollector(object): + """performance metrics collector""" diff --git a/qtip/agent/reporter.py b/qtip/collector/logfile.py index b5c4acfa..6528ea9f 100644 --- a/qtip/agent/reporter.py +++ b/qtip/collector/logfile.py @@ -7,7 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +from base import BaseCollector -class Reporter(object): - """generate test report and push test data to database""" - pass + +class LogfileCollector(BaseCollector): + """collect performance metrics from log files""" diff --git a/qtip/spec/__init__.py b/qtip/driver/__init__.py index e69de29b..e69de29b 100644 --- a/qtip/spec/__init__.py +++ b/qtip/driver/__init__.py diff --git a/qtip/drivers/ansible.py b/qtip/driver/ansible.py index 04e9f9bd..04e9f9bd 100644 --- a/qtip/drivers/ansible.py +++ b/qtip/driver/ansible.py diff --git a/qtip/runner/case.py b/qtip/driver/base.py index eb3febc2..6f5cab3c 100644 --- a/qtip/runner/case.py +++ b/qtip/driver/base.py @@ -7,11 +7,14 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from qtip.base.benchmark import Property -from qtip.spec.metric import MetricSpec +class BaseDriver(object): + """performance testing tool driver""" + def pre_run(self): + pass -class Case(object): - def __init__(self, spec, paths=None): - self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths) - self.config = spec[Property.CONFIG] + def run(self): + pass + + def post_run(self): + pass diff --git a/qtip/driver/sample.py b/qtip/driver/sample.py new file mode 100644 index 00000000..9b347949 --- /dev/null +++ b/qtip/driver/sample.py @@ -0,0 +1,15 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from base import BaseDriver + + +class SampleDriver(BaseDriver): + """sample driver that generates random data for testing""" diff --git a/qtip/drivers/yardstick.py b/qtip/driver/yardstick.py index 83f1b3d8..83f1b3d8 100644 --- a/qtip/drivers/yardstick.py +++ b/qtip/driver/yardstick.py diff --git a/qtip/loader/__init__.py b/qtip/loader/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/loader/__init__.py diff --git a/qtip/base/benchmark.py b/qtip/loader/base.py index b38e6016..f7fcb669 100644 --- a/qtip/base/benchmark.py +++ b/qtip/loader/base.py @@ -12,52 +12,14 @@ from os import listdir from os import path import yaml - -class Property(object): - # list - NAME = 'name' - CONTENT = 'content' - ABSPATH = 'abspath' - # content - TITLE = 'title' - DESCRIPTION = 'description' - # spec - ALGORITHM = 'algorithm' - SECTIONS = 'sections' - WEIGHT = 'weight' - METRICS = 'metrics' - SPEC = 'spec' - WORKLOADS = 'workloads' - # plan - INFO = 'info' - FACILITY = 'facility' - ENGINEER = 'engineer' - SUITES = 'suites' - # suite - QPI_SPEC = 'QPI_spec' - CONDITION = 'condition' - CASES = 'cases' - # case - METRIC_SPEC = 'metric_spec' - CONFIG = 'config' - - -class QtipError(Exception): - pass - - -class Algorithm(object): - ARITHMETIC_MEAN = 'arithmetic mean' - WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean' - GEOMETRIC_MEAN = 'geometric mean' - WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean' - +from qtip.base.error import InvalidFormat, NotFound +from qtip.base.constant import PropName ROOT_DIR = 'benchmarks' -class Benchmark(object): - """Abstract class of QTIP benchmarks""" +class BaseLoader(object): + """Abstract class of QTIP benchmark loader""" DEFAULT_DIR = '.' _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, ROOT_DIR)] @@ -65,7 +27,16 @@ class Benchmark(object): def __init__(self, name, paths=None): self._file = name self._abspath = self._find(name, paths) - self.name = path.splitext(name)[0] + + try: + content = yaml.safe_load(file(self._abspath)) + except yaml.YAMLError: + # TODO(yujunz) log yaml error + raise InvalidFormat(self._abspath) + + self.name = content[PropName.NAME] if PropName.NAME in content \ + else path.splitext(name)[0] + self.content = content def _find(self, name, paths): """find a benchmark in searching paths""" @@ -75,7 +46,7 @@ class Benchmark(object): abspath = path.join(p, name) if path.exists(abspath): return abspath - raise QtipError("'{}' not found in paths: {}".format(name, paths)) + raise NotFound(name, paths) @classmethod def list_all(cls, paths=None): @@ -86,10 +57,6 @@ class Benchmark(object): for name in names: item = cls(name, paths=paths) yield { - Property.NAME: name, - Property.ABSPATH: item._abspath, - Property.CONTENT: item.content()} - - def content(self): - """description of benchmark""" - return yaml.safe_load(file(self._abspath)) + PropName.NAME: name, + PropName.ABSPATH: item._abspath, + PropName.CONTENT: item.content} diff --git a/qtip/spec/metric.py b/qtip/loader/metric.py index e9c70547..d6174e8f 100644 --- a/qtip/spec/metric.py +++ b/qtip/loader/metric.py @@ -7,10 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from qtip.base.benchmark import Benchmark +from base import BaseLoader -class MetricSpec(Benchmark): +class MetricSpec(BaseLoader): """metrics in QTIP are categorized by performance test tools, such as dhrystone, whetstone and etc""" - DEFAULT_DIR = 'metrics' + DEFAULT_DIR = 'metric' diff --git a/qtip/spec/qpi.py b/qtip/loader/qpi.py index b7d7aa02..2972cbd7 100644 --- a/qtip/spec/qpi.py +++ b/qtip/loader/qpi.py @@ -7,10 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from qtip.base.benchmark import Benchmark +from base import BaseLoader -class QPISpec(Benchmark): +class QPISpec(BaseLoader): """ a QPI specification defines how to calculate a performance index from collected metrics. diff --git a/qtip/reporter/__init__.py b/qtip/reporter/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/reporter/__init__.py diff --git a/qtip/reporter/base.py b/qtip/reporter/base.py new file mode 100644 index 00000000..b931d14d --- /dev/null +++ b/qtip/reporter/base.py @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class BaseReporter(object): + """benchmark result reporter""" + def __init__(self, collector=None): + self.collector = collector diff --git a/qtip/agent/collector.py b/qtip/reporter/console.py index 3df21379..136ce358 100644 --- a/qtip/agent/collector.py +++ b/qtip/reporter/console.py @@ -7,7 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +from base import BaseReporter -class Collector(object): - """collect test result and test condition""" - pass + +class ConsoleReporter(BaseReporter): + """report result to console""" diff --git a/qtip/runner/base.py b/qtip/runner/base.py new file mode 100644 index 00000000..07fba104 --- /dev/null +++ b/qtip/runner/base.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.base.constant import PkgName, PropName +from qtip.base.error import NotFound +from qtip.collector.logfile import LogfileCollector +from qtip.driver.sample import SampleDriver +from qtip.loader.base import BaseLoader +from qtip.reporter.console import ConsoleReporter + + +class BaseRunner(BaseLoader): + def __init__(self, name, paths=None, config=None): + super(BaseRunner, self).__init__(name, paths=paths) + if config is None: + config = self.content[PropName.CONFIG] + + driver_name = config[PropName.DRIVER] + collector_name = config[PropName.COLLECTOR] + reporter_name = config[PropName.REPORTER] + + # TODO(yujunz) dynamically load modules by name + + if driver_name == 'sample': + self.driver = SampleDriver() + else: + raise NotFound(driver_name, package=PkgName.DRIVER) + + if collector_name == 'logfile': + self.collector = LogfileCollector() + else: + raise NotFound(collector_name, + package=PkgName.COLLECTOR) + + if reporter_name == 'console': + self.reporter = ConsoleReporter() + else: + raise NotFound(reporter_name, + package=PkgName.REPORTER) diff --git a/qtip/runner/plan.py b/qtip/runner/plan.py index 265ad8d7..f6c1c3bb 100644 --- a/qtip/runner/plan.py +++ b/qtip/runner/plan.py @@ -7,20 +7,19 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from qtip.base.benchmark import Benchmark, Property -from qtip.runner.suite import Suite +from qtip.base.constant import PropName +from qtip.runner.base import BaseRunner +from qtip.loader.qpi import QPISpec -class Plan(Benchmark): +class Plan(BaseRunner): + """ + a benchmark plan is consist of configuration and a QPI list """ - a benchmark plan is consist of basic information and several suites""" - DEFAULT_DIR = 'plans' + DEFAULT_DIR = 'plan' def __init__(self, name, paths=None): - super(Plan, self).__init__(name, paths=paths) - content = self.content() - - self.info = content[Property.INFO] - self.suites = [Suite(suite, paths=paths) - for suite in content[Property.SUITES]] + super(Plan, self).__init__(name, paths) + self.qpis = [QPISpec(qpi, paths=paths) + for qpi in self.content[PropName.QPIS]] diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py deleted file mode 100644 index 55033d2b..00000000 --- a/qtip/runner/suite.py +++ /dev/null @@ -1,22 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from qtip.base.benchmark import Property -from qtip.spec.qpi import QPISpec -from qtip.runner.case import Case - - -class Suite(object): - """a suite of benchmark cases under specified condition""" - def __init__(self, spec, paths=None): - self._paths = paths - self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths) - self.condition = spec.get(Property.CONDITION, {}) - self.cases = [Case(case_spec, paths) - for case_spec in spec.get(Property.CASES, [])] |