diff options
48 files changed, 539 insertions, 336 deletions
@@ -66,3 +66,7 @@ target/ /docs_build/ /docs_output/ .idea + +# setup.py cache +AUTHORS +ChangeLog diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad693d36..a4929172 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,7 @@ Active Reviewers Current list of active reviewers in gerrit group `qtip-reviewers` +* Akhil Batra <akhil.batra@research.iiit.ac.in> * Serena Feng <feng.xiaowei@zte.com.cn> * Taseer Ahmed <taseer94@gmail.com> * Yujun Zhang <zhang.yujunz@zte.com.cn> diff --git a/DEVELOP.md b/DEVELOP.md new file mode 100644 index 00000000..9ec3a93d --- /dev/null +++ b/DEVELOP.md @@ -0,0 +1,101 @@ +# QTIP Developer Guide + +This guide is about how to **develop** QTIP. + +If you just want to use it for performance benchmark, check the user guide +instead. + +## Getting Started + +### Source code + +```bash +~$ git clone https://gerrit.opnfv.org/gerrit/qtip +``` + +### VirtualEnv + +It is recommended to use [virtualenv](https://virtualenv.pypa.io) to isolate +your development environment from system, especially when you are working on +several different projects. + +### Testing + +QTIP use [tox](https://tox.readthedocs.io) to automate the testing tasks + +```bash +$ pip install tox +$ tox +``` + +## Architecture + +**TODO**: move to design spec + +QTIP has a flexible architecture to allow different deployment mode + +- **Standalone**: full feature performance benchmark platform. +- **Agent**: minimal agent driven by external test runners. + +### Standalone Mode (Solo) + +QTIP instance deployed in container, VM or host generate benchmark report and +push data to Indices Hub for storage and visualization. + + + +### Agent Mode (Melody) + +QTIP Collector and Reporter driven by external test framework or runner such as +[yardstick](https://wiki.opnfv.org/display/yardstick), +[pytest](http://doc.pytest.org/) and etc. + + + +## Core Modules + +TBD + +- loader +- runner +- collector +- reporter + +## Drivers + +TBD + +- ansible +- yardstick + +## Interfaces + +### Agent + +TBD + +### CLI + +TBD + +### API + +TBD + +## Assets + +**TODO**: move to user guide + +- benchmark plan +- QPI spec +- metric spec + +## Docker Image + +TBD + +## Annex + +### Directories + +TBD diff --git a/benchmarks/suite/compute.yaml b/benchmarks/suite/compute.yaml index 0761a87b..197d5720 100644 --- a/benchmarks/suite/compute.yaml +++ b/benchmarks/suite/compute.yaml @@ -1,12 +1,12 @@ QPI: compute description: sample performance index of computing -algorithm: weighted arithmetic mean +formula: weighted arithmetic mean section: - name: Integer weight: 0.3 - algorithm: geometric mean + formula: geometric mean perftests: - name: dhrystone workloads: @@ -14,7 +14,7 @@ section: - multi_cpu - name: Floating weight: 0.3 - algorithm: geometric mean + formula: geometric mean perftests: - name: whetstone workloads: @@ -22,7 +22,7 @@ section: - multi_cpu - name: Memory weight: 0.2 - algorithm: geometric mean + formula: geometric mean perftests: - name: ramspeed workloads: @@ -30,7 +30,7 @@ section: - float: [add, average, copy, scale, triad] - name: DPI weight: 0.1 - algorithm: geometric mean + formula: geometric mean perftests: - name: dpi workloads: @@ -38,7 +38,7 @@ section: - pps - name: SSL weight: 0.1 - algorithm: geometric mean + formula: geometric mean perftests: - name: ssl workloads: diff --git a/qtip/base/benchmark.py b/qtip/base/benchmark.py deleted file mode 100644 index b38e6016..00000000 --- a/qtip/base/benchmark.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from itertools import chain -from os import listdir -from os import path -import yaml - - -class Property(object): - # list - NAME = 'name' - CONTENT = 'content' - ABSPATH = 'abspath' - # content - TITLE = 'title' - DESCRIPTION = 'description' - # spec - ALGORITHM = 'algorithm' - SECTIONS = 'sections' - WEIGHT = 'weight' - METRICS = 'metrics' - SPEC = 'spec' - WORKLOADS = 'workloads' - # plan - INFO = 'info' - FACILITY = 'facility' - ENGINEER = 'engineer' - SUITES = 'suites' - # suite - QPI_SPEC = 'QPI_spec' - CONDITION = 'condition' - CASES = 'cases' - # case - METRIC_SPEC = 'metric_spec' - CONFIG = 'config' - - -class QtipError(Exception): - pass - - -class Algorithm(object): - ARITHMETIC_MEAN = 'arithmetic mean' - WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean' - GEOMETRIC_MEAN = 'geometric mean' - WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean' - - -ROOT_DIR = 'benchmarks' - - -class Benchmark(object): - """Abstract class of QTIP benchmarks""" - DEFAULT_DIR = '.' - _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, - ROOT_DIR)] - - def __init__(self, name, paths=None): - self._file = name - self._abspath = self._find(name, paths) - self.name = path.splitext(name)[0] - - def _find(self, name, paths): - """find a benchmark in searching paths""" - paths = self._paths if paths is None else paths - name = path.join(self.DEFAULT_DIR, name) - for p in paths: - abspath = path.join(p, name) - if path.exists(abspath): - return abspath - raise QtipError("'{}' not found in paths: {}".format(name, paths)) - - @classmethod - def list_all(cls, paths=None): - """list all available benchmarks""" - paths = cls._paths if paths is None else paths - names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR)) - for p in paths]) - for name in names: - item = cls(name, paths=paths) - yield { - Property.NAME: name, - Property.ABSPATH: item._abspath, - Property.CONTENT: item.content()} - - def content(self): - """description of benchmark""" - return yaml.safe_load(file(self._abspath)) diff --git a/qtip/base/constant.py b/qtip/base/constant.py new file mode 100644 index 00000000..76481b47 --- /dev/null +++ b/qtip/base/constant.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class FormulaName(object): + """formula names""" + ARITHMETIC_MEAN = 'arithmetic mean' + WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean' + GEOMETRIC_MEAN = 'geometric mean' + WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean' + + +class PkgName(object): + """QTIP package names""" + COLLECTOR = 'collector' + DRIVER = 'driver' + REPORTER = 'reporter' + RUNNER = 'runner' + SPEC = 'spec' + + +class PropName(object): + """property names""" + # list + NAME = 'name' + CONTENT = 'content' + ABSPATH = 'abspath' + # content + name = 'name' + DESCRIPTION = 'description' + # spec + SECTIONS = 'sections' + WEIGHT = 'weight' + FORMULA = 'formula' + METRICS = 'metrics' + WORKLOADS = 'workloads' + # plan + CONFIG = 'config' + FACILITY = 'facility' + ENGINEER = 'engineer' + DRIVER = 'driver' + COLLECTOR = 'collector' + REPORTER = 'reporter' + QPIS = 'QPIs' diff --git a/qtip/base/error.py b/qtip/base/error.py new file mode 100644 index 00000000..01a7f7a6 --- /dev/null +++ b/qtip/base/error.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class QtipError(Exception): + pass + + +class InvalidFormat(QtipError): + def __init__(self, filename): + self.filename = filename + + +class NotFound(QtipError): + def __init__(self, module, package='qtip'): + self.package = package + self.module = module + + +class ToBeDoneError(QtipError): + """something still to be done""" + def __init__(self, method, module): + self.method = method + self.module = module + + +def make_tbd(method, module='qtip'): + def tbd(): + raise ToBeDoneError(method, module) + return tbd diff --git a/qtip/cli/commands/cmd_plan.py b/qtip/cli/commands/cmd_plan.py index 01bf8251..6f622e5a 100644 --- a/qtip/cli/commands/cmd_plan.py +++ b/qtip/cli/commands/cmd_plan.py @@ -9,7 +9,8 @@ import click from prettytable import PrettyTable -from qtip.runner.plan import Plan + +from qtip.loader.plan import Plan @click.group() diff --git a/qtip/drivers/__init__.py b/qtip/collector/__init__.py index e69de29b..e69de29b 100644 --- a/qtip/drivers/__init__.py +++ b/qtip/collector/__init__.py diff --git a/qtip/drivers/base.py b/qtip/collector/base.py index 1aa8d8ad..cd8fc797 100644 --- a/qtip/drivers/base.py +++ b/qtip/collector/base.py @@ -8,5 +8,5 @@ ############################################################################## -class BaseDriver(object): - """performance testing tool driver""" +class BaseCollector(object): + """performance metrics collector""" diff --git a/qtip/agent/reporter.py b/qtip/collector/logfile.py index b5c4acfa..6528ea9f 100644 --- a/qtip/agent/reporter.py +++ b/qtip/collector/logfile.py @@ -7,7 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +from base import BaseCollector -class Reporter(object): - """generate test report and push test data to database""" - pass + +class LogfileCollector(BaseCollector): + """collect performance metrics from log files""" diff --git a/qtip/spec/__init__.py b/qtip/driver/__init__.py index e69de29b..e69de29b 100644 --- a/qtip/spec/__init__.py +++ b/qtip/driver/__init__.py diff --git a/qtip/drivers/ansible.py b/qtip/driver/ansible.py index 04e9f9bd..04e9f9bd 100644 --- a/qtip/drivers/ansible.py +++ b/qtip/driver/ansible.py diff --git a/qtip/spec/qpi.py b/qtip/driver/base.py index b7d7aa02..6f5cab3c 100644 --- a/qtip/spec/qpi.py +++ b/qtip/driver/base.py @@ -7,12 +7,14 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from qtip.base.benchmark import Benchmark +class BaseDriver(object): + """performance testing tool driver""" + def pre_run(self): + pass -class QPISpec(Benchmark): - """ - a QPI specification defines how to calculate a performance index from - collected metrics. - """ - DEFAULT_DIR = 'QPI' + def run(self): + pass + + def post_run(self): + pass diff --git a/qtip/driver/sample.py b/qtip/driver/sample.py new file mode 100644 index 00000000..9b347949 --- /dev/null +++ b/qtip/driver/sample.py @@ -0,0 +1,15 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from base import BaseDriver + + +class SampleDriver(BaseDriver): + """sample driver that generates random data for testing""" diff --git a/qtip/drivers/yardstick.py b/qtip/driver/yardstick.py index 83f1b3d8..83f1b3d8 100644 --- a/qtip/drivers/yardstick.py +++ b/qtip/driver/yardstick.py diff --git a/qtip/loader/__init__.py b/qtip/loader/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/loader/__init__.py diff --git a/qtip/loader/base.py b/qtip/loader/base.py new file mode 100644 index 00000000..2f5ab67a --- /dev/null +++ b/qtip/loader/base.py @@ -0,0 +1,61 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from itertools import chain +from os import listdir +from os import path +import yaml + +from qtip.base.error import InvalidFormat, NotFound +from qtip.base.constant import PropName + +ROOT_DIR = 'benchmarks' + + +class BaseLoader(object): + """Abstract class of QTIP benchmark loader""" + RELATIVE_PATH = '.' + _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, + ROOT_DIR)] + + def __init__(self, name, paths=None): + self._file = name + self._abspath = self._find(name, paths=paths) + + try: + content = yaml.safe_load(file(self._abspath)) + except yaml.YAMLError: + # TODO(yujunz) log yaml error + raise InvalidFormat(self._abspath) + + self.name = content[PropName.NAME] if PropName.NAME in content \ + else path.splitext(name)[0] + self.content = content + + def _find(self, name, paths=None): + """find a benchmark in searching paths""" + paths = self._paths if paths is None else paths + for p in paths: + abspath = path.join(p, self.RELATIVE_PATH, name) + if path.exists(abspath): + return abspath + raise NotFound(name, paths) + + @classmethod + def list_all(cls, paths=None): + """list all available benchmarks""" + paths = cls._paths if paths is None else paths + names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH)) + for p in paths]) + for name in names: + item = cls(name, paths=paths) + yield { + PropName.NAME: name, + PropName.ABSPATH: item._abspath, + PropName.CONTENT: item.content} diff --git a/qtip/spec/metric.py b/qtip/loader/metric.py index e9c70547..8b6fa5d3 100644 --- a/qtip/spec/metric.py +++ b/qtip/loader/metric.py @@ -7,10 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from qtip.base.benchmark import Benchmark +from base import BaseLoader -class MetricSpec(Benchmark): +class MetricSpec(BaseLoader): """metrics in QTIP are categorized by performance test tools, such as dhrystone, whetstone and etc""" - DEFAULT_DIR = 'metrics' + RELATIVE_PATH = 'metric' diff --git a/qtip/runner/plan.py b/qtip/loader/plan.py index 265ad8d7..cf517ea2 100644 --- a/qtip/runner/plan.py +++ b/qtip/loader/plan.py @@ -7,20 +7,21 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from qtip.base.benchmark import Benchmark, Property -from qtip.runner.suite import Suite +from qtip.base.constant import PropName +from qtip.loader.base import BaseLoader +from qtip.loader.qpi import QPISpec -class Plan(Benchmark): + +class Plan(BaseLoader): + """ + a benchmark plan is consist of configuration and a QPI list """ - a benchmark plan is consist of basic information and several suites""" - DEFAULT_DIR = 'plans' + RELATIVE_PATH = 'plan' def __init__(self, name, paths=None): - super(Plan, self).__init__(name, paths=paths) - content = self.content() + super(Plan, self).__init__(name, paths) - self.info = content[Property.INFO] - self.suites = [Suite(suite, paths=paths) - for suite in content[Property.SUITES]] + self.qpis = [QPISpec(qpi, paths=paths) + for qpi in self.content[PropName.QPIS]] diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py new file mode 100644 index 00000000..cfa918c5 --- /dev/null +++ b/qtip/loader/qpi.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from base import BaseLoader +from metric import MetricSpec + +from qtip.base.constant import PropName +from qtip.utils.formula import Formula + + +class QPISpec(BaseLoader): + """ + a QPI specification defines how to calculate a performance index from + collected metrics. + """ + RELATIVE_PATH = 'QPI' + + def __init__(self, name, paths=None): + super(QPISpec, self).__init__(name, paths=paths) + content = self.content + self.formula = Formula(content[PropName.FORMULA]) + self.sections = [Section(record, paths=paths) + for record in content[PropName.SECTIONS]] + + +class Section(object): + def __init__(self, content, paths=None): + self.name = content[PropName.NAME] + self.weight = content[PropName.WEIGHT] + self.formula = Formula(content[PropName.FORMULA]) + self.metrics = [MetricSpec(record, paths=paths) + for record in content[PropName.METRICS]] diff --git a/qtip/reporter/__init__.py b/qtip/reporter/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/reporter/__init__.py diff --git a/qtip/reporter/base.py b/qtip/reporter/base.py new file mode 100644 index 00000000..b931d14d --- /dev/null +++ b/qtip/reporter/base.py @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class BaseReporter(object): + """benchmark result reporter""" + def __init__(self, collector=None): + self.collector = collector diff --git a/qtip/agent/collector.py b/qtip/reporter/console.py index 3df21379..136ce358 100644 --- a/qtip/agent/collector.py +++ b/qtip/reporter/console.py @@ -7,7 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +from base import BaseReporter -class Collector(object): - """collect test result and test condition""" - pass + +class ConsoleReporter(BaseReporter): + """report result to console""" diff --git a/qtip/runner/__init__.py b/qtip/runner/__init__.py index e69de29b..eab81156 100644 --- a/qtip/runner/__init__.py +++ b/qtip/runner/__init__.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.base.constant import PkgName, PropName +from qtip.base.error import NotFound +from qtip.collector.stdout import StdoutCollector +from qtip.driver.random import RandomDriver +from qtip.reporter.console import ConsoleReporter + + +class Runner(object): + def __init__(self, spec, config=None): + if config is None: + config = spec[PropName.CONFIG] + + driver_name = config[PropName.DRIVER] + collector_name = config[PropName.COLLECTOR] + reporter_name = config[PropName.REPORTER] + + # TODO(yujunz) dynamically load modules by name + + if driver_name == 'random': + self.driver = RandomDriver() + else: + raise NotFound(driver_name, package=PkgName.DRIVER) + + if collector_name == 'stdout': + self.collector = StdoutCollector() + else: + raise NotFound(collector_name, + package=PkgName.COLLECTOR) + + if reporter_name == 'console': + self.reporter = ConsoleReporter() + else: + raise NotFound(reporter_name, + package=PkgName.REPORTER) diff --git a/qtip/runner/case.py b/qtip/runner/case.py deleted file mode 100644 index eb3febc2..00000000 --- a/qtip/runner/case.py +++ /dev/null @@ -1,17 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from qtip.base.benchmark import Property -from qtip.spec.metric import MetricSpec - - -class Case(object): - def __init__(self, spec, paths=None): - self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths) - self.config = spec[Property.CONFIG] diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py deleted file mode 100644 index 55033d2b..00000000 --- a/qtip/runner/suite.py +++ /dev/null @@ -1,22 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from qtip.base.benchmark import Property -from qtip.spec.qpi import QPISpec -from qtip.runner.case import Case - - -class Suite(object): - """a suite of benchmark cases under specified condition""" - def __init__(self, spec, paths=None): - self._paths = paths - self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths) - self.condition = spec.get(Property.CONDITION, {}) - self.cases = [Case(case_spec, paths) - for case_spec in spec.get(Property.CASES, [])] diff --git a/qtip/utils/formula.py b/qtip/utils/formula.py new file mode 100644 index 00000000..cdfbae86 --- /dev/null +++ b/qtip/utils/formula.py @@ -0,0 +1,29 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import numpy + +from qtip.base.error import make_tbd +from qtip.base.constant import FormulaName + + +MAPPING = { + FormulaName.ARITHMETIC_MEAN: numpy.mean, + FormulaName.WEIGHTED_ARITHMETIC_MEAN: numpy.average, + # TODO(yujunz) find or implement the method + FormulaName.GEOMETRIC_MEAN: make_tbd(FormulaName.GEOMETRIC_MEAN, __name__), + # TODO(yujunz) find or implement the method + FormulaName.WEIGHTED_GEOMETRIC_MEAN: + make_tbd(FormulaName.GEOMETRIC_MEAN, __name__)} + + +class Formula: + """calculate a score from give data""" + def __init__(self, name): + self.calculate = MAPPING[name] diff --git a/requirements.txt b/requirements.txt index cdaeef70..ec566b83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ Flask==0.11.1 Flask-RESTful==0.3.5 flask-restful-swagger==0.19 ansible==2.1.1.0 +numpy==1.11.3 diff --git a/tests/conftest.py b/tests/conftest.py index 8e2ecf7c..7acb75e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,8 +8,11 @@ ############################################################################## from os import path + import pytest +from qtip.loader.plan import Plan + @pytest.fixture(scope='session') def data_root(): @@ -19,3 +22,8 @@ def data_root(): @pytest.fixture(scope='session') def benchmarks_root(data_root): return path.join(data_root, 'benchmarks') + + +@pytest.fixture(scope='session') +def plan(benchmarks_root): + return Plan('fake-plan.yaml', [benchmarks_root]) diff --git a/tests/data/benchmarks/QPI/compute.yaml b/tests/data/benchmarks/QPI/compute.yaml index 5ad8a4b6..e28efaf9 100644 --- a/tests/data/benchmarks/QPI/compute.yaml +++ b/tests/data/benchmarks/QPI/compute.yaml @@ -1,44 +1,29 @@ title: compute description: sample performance index of computing -algorithm: weighted arithmetic mean +formula: weighted arithmetic mean sections: - name: Integer weight: 0.3 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dhrystone.yaml - workloads: - - single_cpu - - multi_cpu + - dhrystone.yaml - name: Float weight: 0.3 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dhrystone.yaml - workloads: - - single_cpu - - multi_cpu + - whetstone.yaml - name: Memory weight: 0.2 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: ramspeed.yaml - workloads: - - int: [add, average, copy, scale, triad] - - float: [add, average, copy, scale, triad] + - ramspeed.yaml - name: DPI weight: 0.1 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dpi.yaml - workloads: - - bps - - pps + - dpi.yaml - name: SSL weight: 0.1 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: ssl.yaml - workloads: - - aes_128_cbc: [512, 1024, 2048, 4096] - - rsa_sig: [16, 64, 256, 1024, 8192] + - ssl.yaml diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml new file mode 100644 index 00000000..aa1097f4 --- /dev/null +++ b/tests/data/benchmarks/QPI/fake-qpi.yaml @@ -0,0 +1,9 @@ +name: Fake QPI +description: a fake QPI producing random result +formula: weighted arithmetic mean +sections: +- name: Fake Section + weight: 0.5 + formula: geometric mean + metrics: + - fake-metric.yaml diff --git a/tests/data/benchmarks/metric/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml new file mode 100644 index 00000000..220b7841 --- /dev/null +++ b/tests/data/benchmarks/metric/dhrystone.yaml @@ -0,0 +1,9 @@ +name: dhrystone +description: > + A synthetic computing benchmark program intended to be representative of + system (integer) programming. +links: + - https://en.wikipedia.org/wiki/Dhrystone +workloads: + - single_cpu + - multi_cpu diff --git a/tests/data/benchmarks/metrics/dpi.yaml b/tests/data/benchmarks/metric/dpi.yaml index fc24c8d1..fc24c8d1 100644 --- a/tests/data/benchmarks/metrics/dpi.yaml +++ b/tests/data/benchmarks/metric/dpi.yaml diff --git a/tests/data/benchmarks/metric/fake-metric.yaml b/tests/data/benchmarks/metric/fake-metric.yaml new file mode 100644 index 00000000..c5fcc5c2 --- /dev/null +++ b/tests/data/benchmarks/metric/fake-metric.yaml @@ -0,0 +1,7 @@ +name: fake compute +description: > + a fake compute performance index which generate random result +workloads: # all supported workloads + - fake_workload_1 + - fake_workload_2 + - fake_workload_3 diff --git a/tests/data/benchmarks/metrics/ramspeed.yaml b/tests/data/benchmarks/metric/ramspeed.yaml index bb7618b7..bb7618b7 100644 --- a/tests/data/benchmarks/metrics/ramspeed.yaml +++ b/tests/data/benchmarks/metric/ramspeed.yaml diff --git a/tests/data/benchmarks/metrics/ssl.yaml b/tests/data/benchmarks/metric/ssl.yaml index 21e8add7..21e8add7 100644 --- a/tests/data/benchmarks/metrics/ssl.yaml +++ b/tests/data/benchmarks/metric/ssl.yaml diff --git a/tests/data/benchmarks/metric/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml new file mode 100644 index 00000000..448c9645 --- /dev/null +++ b/tests/data/benchmarks/metric/whetstone.yaml @@ -0,0 +1,10 @@ +name: whetstone +description: > + A synthetic benchmark for evaluating the performance of computers. + The Whetstone benchmark primarily measures the floating-point arithmetic + performance. +links: + - https://en.wikipedia.org/wiki/Whetstone_(benchmark) +workloads: + - single_cpu + - multi_cpu diff --git a/tests/data/benchmarks/metrics/dhrystone.yaml b/tests/data/benchmarks/metrics/dhrystone.yaml deleted file mode 100644 index b0d55ed2..00000000 --- a/tests/data/benchmarks/metrics/dhrystone.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: dhrystone -description: > - a synthetic computing benchmark program intended to be representative of - system (integer) programming -workloads: - - single_cpu - - multi_cpu diff --git a/tests/data/benchmarks/metrics/whetstone.yaml b/tests/data/benchmarks/metrics/whetstone.yaml deleted file mode 100644 index d83680c4..00000000 --- a/tests/data/benchmarks/metrics/whetstone.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: dhrystone -description: a synthetic benchmark for evaluating the performance of computers -workloads: - - single_cpu - - multi_cpu diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml new file mode 100644 index 00000000..8887f66d --- /dev/null +++ b/tests/data/benchmarks/plan/fake-plan.yaml @@ -0,0 +1,10 @@ +name: fake plan +description: fake benchmark plan for demonstration and testing +config: + facility: local + engineer: local + driver: sample + collector: logfile + reporter: console +QPIs: + - fake-qpi.yaml diff --git a/tests/data/benchmarks/plans/verification.yaml b/tests/data/benchmarks/plans/verification.yaml deleted file mode 100644 index b146ee37..00000000 --- a/tests/data/benchmarks/plans/verification.yaml +++ /dev/null @@ -1,21 +0,0 @@ -title: verification -description: benchmark plan for QTIP verification -info: - facility: opnfv-ci - engineer: opnfv-bot -suites: - - QPI_spec: compute.yaml -# Uncomment next line to setup suite running condition -# condition: {} -# Uncomment the following lines setup case running configuration -# cases: -# - metric_spec: dhrystone.yaml -# config: {} -# - metric_spec: whetstone.yaml -# config: {} -# - metric_spec: ramspeed.yaml -# config: {} -# - metric_spec: dpi.yaml -# config: {} -# - metric_spec: ssl.yaml -# config: {} diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py index 5eced700..d2be0388 100644 --- a/tests/unit/loader/metric_test.py +++ b/tests/unit/loader/metric_test.py @@ -9,8 +9,8 @@ import pytest -from qtip.base.benchmark import Property -from qtip.spec.metric import MetricSpec +from qtip.base.constant import PropName +from qtip.loader.metric import MetricSpec @pytest.fixture(scope='module') @@ -29,17 +29,17 @@ def init_test(metric_spec): def list_all_test(): metric_list = MetricSpec.list_all() - assert len(list(metric_list)) is 1 + assert len(list(metric_list)) is 6 for desc in metric_list: - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc - assert Property.ABSPATH is not None + assert PropName.NAME in desc + assert PropName.DESCRIPTION in desc + assert PropName.ABSPATH in desc + assert PropName.ABSPATH is not None def content_test(metric): - content = metric.content() - assert Property.NAME in content - assert Property.DESCRIPTION in content - assert Property.WORKLOADS in content - assert isinstance(content[Property.WORKLOADS], list) + content = metric.content + assert PropName.NAME in content + assert PropName.DESCRIPTION in content + assert PropName.WORKLOADS in content + assert isinstance(content[PropName.WORKLOADS], list) diff --git a/tests/unit/runner/plan_test.py b/tests/unit/loader/plan_test.py index d783e5e9..6aab5e8a 100644 --- a/tests/unit/runner/plan_test.py +++ b/tests/unit/loader/plan_test.py @@ -9,12 +9,15 @@ import pytest -from qtip.base.benchmark import Property -from qtip.runner.plan import Plan +from qtip.base.constant import PropName +from qtip.loader.plan import Plan, QPISpec def test_init(plan): - assert plan.name == 'verification' + assert plan.name == 'fake plan' + assert isinstance(plan.content, dict) + for qpi in plan.qpis: + assert isinstance(qpi, QPISpec) with pytest.raises(TypeError) as excinfo: Plan() @@ -26,13 +29,15 @@ def test_list_all(benchmarks_root): plan_list = Plan.list_all(paths=[benchmarks_root]) assert len(list(plan_list)) is 1 for desc in plan_list: - assert Property.NAME in desc - assert Property.CONTENT in desc - assert Property.ABSPATH in desc - assert Property.ABSPATH is not None + assert PropName.NAME in desc + assert PropName.CONTENT in desc + assert PropName.ABSPATH in desc + assert PropName.ABSPATH is not None def test_content(plan): - content = plan.content() - assert Property.TITLE in content - assert Property.DESCRIPTION in content + content = plan.content + assert PropName.NAME in content + assert PropName.DESCRIPTION in content + assert PropName.CONFIG in content + assert PropName.QPIS in content diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py index bfa1f580..4b3fd4d0 100644 --- a/tests/unit/loader/qpi_test.py +++ b/tests/unit/loader/qpi_test.py @@ -9,8 +9,8 @@ import pytest -from qtip.base.benchmark import Algorithm, Property -from qtip.spec.qpi import QPISpec +from qtip.base.constant import FormulaName, PropName +from qtip.loader.qpi import QPISpec QPI_SPEC = 'compute.yaml' @@ -31,22 +31,22 @@ def test_init(qpi_spec): def test_list_all(benchmarks_root): qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root]) - assert len(list(qpi_spec_list)) is 1 + assert len(list(qpi_spec_list)) is 2 for item in qpi_spec_list: - assert Property.NAME in item - assert Property.CONTENT in item - assert Property.ABSPATH in item - assert Property.ABSPATH is not None + assert PropName.NAME in item + assert PropName.CONTENT in item + assert PropName.ABSPATH in item + assert PropName.ABSPATH is not None def test_content(qpi_spec): - content = qpi_spec.content() - assert Property.DESCRIPTION in content - assert Property.ALGORITHM in content - assert Property.SECTIONS in content + content = qpi_spec.content + assert PropName.DESCRIPTION in content + assert PropName.FORMULA in content + assert PropName.SECTIONS in content - assert content[Property.ALGORITHM] in Algorithm.__dict__.values() - sections = content[Property.SECTIONS] + assert content[PropName.FORMULA] in FormulaName.__dict__.values() + sections = content[PropName.SECTIONS] assert isinstance(sections, list) for section in sections: - assert Property.NAME in section + assert PropName.NAME in section diff --git a/tests/unit/runner/case_test.py b/tests/unit/runner/case_test.py deleted file mode 100644 index 59a54a84..00000000 --- a/tests/unit/runner/case_test.py +++ /dev/null @@ -1,15 +0,0 @@ -############################################################### -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from qtip.spec.metric import MetricSpec - - -def init_test(case): - assert isinstance(case.metric_spec, MetricSpec) - assert isinstance(case.config, dict) diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py deleted file mode 100644 index 6d14f7ae..00000000 --- a/tests/unit/runner/conftest.py +++ /dev/null @@ -1,30 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from qtip.base.benchmark import Property -from qtip.runner.case import Case -from qtip.runner.plan import Plan -from qtip.runner.suite import Suite - - -@pytest.fixture(scope='module') -def plan(benchmarks_root): - return Plan('verification.yaml', paths=[benchmarks_root]) - - -@pytest.fixture(scope='module') -def suite(plan): - return Suite(plan[Property.SUITES][0]) - - -@pytest.fixture(scope='module') -def case(suite): - return Case(suite[Property.CASES][0]) diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py deleted file mode 100644 index 7dad8f62..00000000 --- a/tests/unit/runner/suite_test.py +++ /dev/null @@ -1,27 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from qtip.runner.suite import Suite -from qtip.runner.case import Case -from qtip.spec.qpi import QPISpec - - -def init_test(suite): - assert isinstance(suite.qpi, QPISpec) - assert isinstance(suite.condition, dict) - assert isinstance(suite.cases, list) - for case in suite.cases: - assert isinstance(case, Case) - - with pytest.raises(TypeError) as excinfo: - Suite() - assert '__init__() takes exactly 2 arguments (1 given)' \ - in str(excinfo.value) |