diff options
| -rw-r--r-- | benchmarks/suite/compute.yaml | 12 | ||||
| -rw-r--r-- | qtip/base/constant.py | 6 | ||||
| -rw-r--r-- | qtip/base/error.py | 13 | ||||
| -rw-r--r-- | qtip/cli/commands/cmd_plan.py | 3 | ||||
| -rw-r--r-- | qtip/loader/base.py | 11 | ||||
| -rw-r--r-- | qtip/loader/metric.py | 2 | ||||
| -rw-r--r-- | qtip/loader/plan.py (renamed from qtip/runner/plan.py) | 8 | ||||
| -rw-r--r-- | qtip/loader/qpi.py | 22 | ||||
| -rw-r--r-- | qtip/runner/__init__.py | 43 | ||||
| -rw-r--r-- | qtip/runner/base.py | 45 | ||||
| -rw-r--r-- | qtip/utils/formula.py | 29 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | tests/conftest.py | 3 | ||||
| -rw-r--r-- | tests/data/benchmarks/QPI/compute.yaml | 37 | ||||
| -rw-r--r-- | tests/data/benchmarks/QPI/fake-qpi.yaml | 6 | ||||
| -rw-r--r-- | tests/data/benchmarks/metric/dhrystone.yaml | 6 | ||||
| -rw-r--r-- | tests/data/benchmarks/metric/whetstone.yaml | 9 | ||||
| -rw-r--r-- | tests/unit/loader/plan_test.py (renamed from tests/unit/runner/plan_test.py) | 7 | ||||
| -rw-r--r-- | tests/unit/loader/qpi_test.py | 6 |
19 files changed, 165 insertions, 104 deletions
diff --git a/benchmarks/suite/compute.yaml b/benchmarks/suite/compute.yaml index 0761a87b..197d5720 100644 --- a/benchmarks/suite/compute.yaml +++ b/benchmarks/suite/compute.yaml @@ -1,12 +1,12 @@ QPI: compute description: sample performance index of computing -algorithm: weighted arithmetic mean +formula: weighted arithmetic mean section: - name: Integer weight: 0.3 - algorithm: geometric mean + formula: geometric mean perftests: - name: dhrystone workloads: @@ -14,7 +14,7 @@ section: - multi_cpu - name: Floating weight: 0.3 - algorithm: geometric mean + formula: geometric mean perftests: - name: whetstone workloads: @@ -22,7 +22,7 @@ section: - multi_cpu - name: Memory weight: 0.2 - algorithm: geometric mean + formula: geometric mean perftests: - name: ramspeed workloads: @@ -30,7 +30,7 @@ section: - float: [add, average, copy, scale, triad] - name: DPI weight: 0.1 - algorithm: geometric mean + formula: geometric mean perftests: - name: dpi workloads: @@ -38,7 +38,7 @@ section: - pps - name: SSL weight: 0.1 - algorithm: geometric mean + formula: geometric mean perftests: - name: ssl workloads: diff --git a/qtip/base/constant.py b/qtip/base/constant.py index 187f0706..76481b47 100644 --- a/qtip/base/constant.py +++ b/qtip/base/constant.py @@ -8,8 +8,8 @@ ############################################################################## -class AlgoName(object): - """algorithm names""" +class FormulaName(object): + """formula names""" ARITHMETIC_MEAN = 'arithmetic mean' WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean' GEOMETRIC_MEAN = 'geometric mean' @@ -37,7 +37,7 @@ class PropName(object): # spec SECTIONS = 'sections' WEIGHT = 'weight' - ALGORITHM = 'algorithm' + FORMULA = 'formula' METRICS = 'metrics' WORKLOADS = 'workloads' # plan diff --git a/qtip/base/error.py b/qtip/base/error.py index d364c532..01a7f7a6 100644 --- a/qtip/base/error.py +++ b/qtip/base/error.py @@ -21,3 +21,16 @@ class NotFound(QtipError): def __init__(self, module, package='qtip'): self.package = package self.module = module + + +class ToBeDoneError(QtipError): + """something still to be done""" + def __init__(self, method, module): + self.method = method + self.module = module + + +def make_tbd(method, module='qtip'): + def tbd(): + raise ToBeDoneError(method, module) + return tbd diff --git a/qtip/cli/commands/cmd_plan.py b/qtip/cli/commands/cmd_plan.py index 01bf8251..6f622e5a 100644 --- a/qtip/cli/commands/cmd_plan.py +++ b/qtip/cli/commands/cmd_plan.py @@ -9,7 +9,8 @@ import click from prettytable import PrettyTable -from qtip.runner.plan import Plan + +from qtip.loader.plan import Plan @click.group() diff --git a/qtip/loader/base.py b/qtip/loader/base.py index f7fcb669..2f5ab67a 100644 --- a/qtip/loader/base.py +++ b/qtip/loader/base.py @@ -20,13 +20,13 @@ ROOT_DIR = 'benchmarks' class BaseLoader(object): """Abstract class of QTIP benchmark loader""" - DEFAULT_DIR = '.' + RELATIVE_PATH = '.' _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, ROOT_DIR)] def __init__(self, name, paths=None): self._file = name - self._abspath = self._find(name, paths) + self._abspath = self._find(name, paths=paths) try: content = yaml.safe_load(file(self._abspath)) @@ -38,12 +38,11 @@ class BaseLoader(object): else path.splitext(name)[0] self.content = content - def _find(self, name, paths): + def _find(self, name, paths=None): """find a benchmark in searching paths""" paths = self._paths if paths is None else paths - name = path.join(self.DEFAULT_DIR, name) for p in paths: - abspath = path.join(p, name) + abspath = path.join(p, self.RELATIVE_PATH, name) if path.exists(abspath): return abspath raise NotFound(name, paths) @@ -52,7 +51,7 @@ class BaseLoader(object): def list_all(cls, paths=None): """list all available benchmarks""" paths = cls._paths if paths is None else paths - names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR)) + names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH)) for p in paths]) for name in names: item = cls(name, paths=paths) diff --git a/qtip/loader/metric.py b/qtip/loader/metric.py index d6174e8f..8b6fa5d3 100644 --- a/qtip/loader/metric.py +++ b/qtip/loader/metric.py @@ -13,4 +13,4 @@ from base import BaseLoader class MetricSpec(BaseLoader): """metrics in QTIP are categorized by performance test tools, such as dhrystone, whetstone and etc""" - DEFAULT_DIR = 'metric' + RELATIVE_PATH = 'metric' diff --git a/qtip/runner/plan.py b/qtip/loader/plan.py index f6c1c3bb..cf517ea2 100644 --- a/qtip/runner/plan.py +++ b/qtip/loader/plan.py @@ -7,19 +7,21 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## + from qtip.base.constant import PropName -from qtip.runner.base import BaseRunner +from qtip.loader.base import BaseLoader from qtip.loader.qpi import QPISpec -class Plan(BaseRunner): +class Plan(BaseLoader): """ a benchmark plan is consist of configuration and a QPI list """ - DEFAULT_DIR = 'plan' + RELATIVE_PATH = 'plan' def __init__(self, name, paths=None): super(Plan, self).__init__(name, paths) + self.qpis = [QPISpec(qpi, paths=paths) for qpi in self.content[PropName.QPIS]] diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py index 2972cbd7..cfa918c5 100644 --- a/qtip/loader/qpi.py +++ b/qtip/loader/qpi.py @@ -8,6 +8,10 @@ ############################################################################## from base import BaseLoader +from metric import MetricSpec + +from qtip.base.constant import PropName +from qtip.utils.formula import Formula class QPISpec(BaseLoader): @@ -15,4 +19,20 @@ class QPISpec(BaseLoader): a QPI specification defines how to calculate a performance index from collected metrics. """ - DEFAULT_DIR = 'QPI' + RELATIVE_PATH = 'QPI' + + def __init__(self, name, paths=None): + super(QPISpec, self).__init__(name, paths=paths) + content = self.content + self.formula = Formula(content[PropName.FORMULA]) + self.sections = [Section(record, paths=paths) + for record in content[PropName.SECTIONS]] + + +class Section(object): + def __init__(self, content, paths=None): + self.name = content[PropName.NAME] + self.weight = content[PropName.WEIGHT] + self.formula = Formula(content[PropName.FORMULA]) + self.metrics = [MetricSpec(record, paths=paths) + for record in content[PropName.METRICS]] diff --git a/qtip/runner/__init__.py b/qtip/runner/__init__.py index e69de29b..eab81156 100644 --- a/qtip/runner/__init__.py +++ b/qtip/runner/__init__.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.base.constant import PkgName, PropName +from qtip.base.error import NotFound +from qtip.collector.stdout import StdoutCollector +from qtip.driver.random import RandomDriver +from qtip.reporter.console import ConsoleReporter + + +class Runner(object): + def __init__(self, spec, config=None): + if config is None: + config = spec[PropName.CONFIG] + + driver_name = config[PropName.DRIVER] + collector_name = config[PropName.COLLECTOR] + reporter_name = config[PropName.REPORTER] + + # TODO(yujunz) dynamically load modules by name + + if driver_name == 'random': + self.driver = RandomDriver() + else: + raise NotFound(driver_name, package=PkgName.DRIVER) + + if collector_name == 'stdout': + self.collector = StdoutCollector() + else: + raise NotFound(collector_name, + package=PkgName.COLLECTOR) + + if reporter_name == 'console': + self.reporter = ConsoleReporter() + else: + raise NotFound(reporter_name, + package=PkgName.REPORTER) diff --git a/qtip/runner/base.py b/qtip/runner/base.py deleted file mode 100644 index 07fba104..00000000 --- a/qtip/runner/base.py +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from qtip.base.constant import PkgName, PropName -from qtip.base.error import NotFound -from qtip.collector.logfile import LogfileCollector -from qtip.driver.sample import SampleDriver -from qtip.loader.base import BaseLoader -from qtip.reporter.console import ConsoleReporter - - -class BaseRunner(BaseLoader): - def __init__(self, name, paths=None, config=None): - super(BaseRunner, self).__init__(name, paths=paths) - if config is None: - config = self.content[PropName.CONFIG] - - driver_name = config[PropName.DRIVER] - collector_name = config[PropName.COLLECTOR] - reporter_name = config[PropName.REPORTER] - - # TODO(yujunz) dynamically load modules by name - - if driver_name == 'sample': - self.driver = SampleDriver() - else: - raise NotFound(driver_name, package=PkgName.DRIVER) - - if collector_name == 'logfile': - self.collector = LogfileCollector() - else: - raise NotFound(collector_name, - package=PkgName.COLLECTOR) - - if reporter_name == 'console': - self.reporter = ConsoleReporter() - else: - raise NotFound(reporter_name, - package=PkgName.REPORTER) diff --git a/qtip/utils/formula.py b/qtip/utils/formula.py new file mode 100644 index 00000000..cdfbae86 --- /dev/null +++ b/qtip/utils/formula.py @@ -0,0 +1,29 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import numpy + +from qtip.base.error import make_tbd +from qtip.base.constant import FormulaName + + +MAPPING = { + FormulaName.ARITHMETIC_MEAN: numpy.mean, + FormulaName.WEIGHTED_ARITHMETIC_MEAN: numpy.average, + # TODO(yujunz) find or implement the method + FormulaName.GEOMETRIC_MEAN: make_tbd(FormulaName.GEOMETRIC_MEAN, __name__), + # TODO(yujunz) find or implement the method + FormulaName.WEIGHTED_GEOMETRIC_MEAN: + make_tbd(FormulaName.GEOMETRIC_MEAN, __name__)} + + +class Formula: + """calculate a score from give data""" + def __init__(self, name): + self.calculate = MAPPING[name] diff --git a/requirements.txt b/requirements.txt index cdaeef70..ec566b83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ Flask==0.11.1 Flask-RESTful==0.3.5 flask-restful-swagger==0.19 ansible==2.1.1.0 +numpy==1.11.3 diff --git a/tests/conftest.py b/tests/conftest.py index f1ec91fb..7acb75e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,9 +8,10 @@ ############################################################################## from os import path + import pytest -from qtip.runner.plan import Plan +from qtip.loader.plan import Plan @pytest.fixture(scope='session') diff --git a/tests/data/benchmarks/QPI/compute.yaml b/tests/data/benchmarks/QPI/compute.yaml index 5ad8a4b6..e28efaf9 100644 --- a/tests/data/benchmarks/QPI/compute.yaml +++ b/tests/data/benchmarks/QPI/compute.yaml @@ -1,44 +1,29 @@ title: compute description: sample performance index of computing -algorithm: weighted arithmetic mean +formula: weighted arithmetic mean sections: - name: Integer weight: 0.3 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dhrystone.yaml - workloads: - - single_cpu - - multi_cpu + - dhrystone.yaml - name: Float weight: 0.3 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dhrystone.yaml - workloads: - - single_cpu - - multi_cpu + - whetstone.yaml - name: Memory weight: 0.2 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: ramspeed.yaml - workloads: - - int: [add, average, copy, scale, triad] - - float: [add, average, copy, scale, triad] + - ramspeed.yaml - name: DPI weight: 0.1 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dpi.yaml - workloads: - - bps - - pps + - dpi.yaml - name: SSL weight: 0.1 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: ssl.yaml - workloads: - - aes_128_cbc: [512, 1024, 2048, 4096] - - rsa_sig: [16, 64, 256, 1024, 8192] + - ssl.yaml diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml index d75c6568..aa1097f4 100644 --- a/tests/data/benchmarks/QPI/fake-qpi.yaml +++ b/tests/data/benchmarks/QPI/fake-qpi.yaml @@ -1,9 +1,9 @@ name: Fake QPI description: a fake QPI producing random result -algorithm: weighted arithmetic mean +formula: weighted arithmetic mean sections: - name: Fake Section weight: 0.5 - algorithm: geometric mean + formula: geometric mean metrics: - - fake_metric.yaml + - fake-metric.yaml diff --git a/tests/data/benchmarks/metric/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml index b0d55ed2..220b7841 100644 --- a/tests/data/benchmarks/metric/dhrystone.yaml +++ b/tests/data/benchmarks/metric/dhrystone.yaml @@ -1,7 +1,9 @@ name: dhrystone description: > - a synthetic computing benchmark program intended to be representative of - system (integer) programming + A synthetic computing benchmark program intended to be representative of + system (integer) programming. +links: + - https://en.wikipedia.org/wiki/Dhrystone workloads: - single_cpu - multi_cpu diff --git a/tests/data/benchmarks/metric/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml index d83680c4..448c9645 100644 --- a/tests/data/benchmarks/metric/whetstone.yaml +++ b/tests/data/benchmarks/metric/whetstone.yaml @@ -1,5 +1,10 @@ -name: dhrystone -description: a synthetic benchmark for evaluating the performance of computers +name: whetstone +description: > + A synthetic benchmark for evaluating the performance of computers. + The Whetstone benchmark primarily measures the floating-point arithmetic + performance. +links: + - https://en.wikipedia.org/wiki/Whetstone_(benchmark) workloads: - single_cpu - multi_cpu diff --git a/tests/unit/runner/plan_test.py b/tests/unit/loader/plan_test.py index 7b3611d1..6aab5e8a 100644 --- a/tests/unit/runner/plan_test.py +++ b/tests/unit/loader/plan_test.py @@ -10,11 +10,14 @@ import pytest from qtip.base.constant import PropName -from qtip.runner.plan import Plan +from qtip.loader.plan import Plan, QPISpec def test_init(plan): assert plan.name == 'fake plan' + assert isinstance(plan.content, dict) + for qpi in plan.qpis: + assert isinstance(qpi, QPISpec) with pytest.raises(TypeError) as excinfo: Plan() @@ -36,3 +39,5 @@ def test_content(plan): content = plan.content assert PropName.NAME in content assert PropName.DESCRIPTION in content + assert PropName.CONFIG in content + assert PropName.QPIS in content diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py index c0d4b377..4b3fd4d0 100644 --- a/tests/unit/loader/qpi_test.py +++ b/tests/unit/loader/qpi_test.py @@ -9,7 +9,7 @@ import pytest -from qtip.base.constant import AlgoName, PropName +from qtip.base.constant import FormulaName, PropName from qtip.loader.qpi import QPISpec QPI_SPEC = 'compute.yaml' @@ -42,10 +42,10 @@ def test_list_all(benchmarks_root): def test_content(qpi_spec): content = qpi_spec.content assert PropName.DESCRIPTION in content - assert PropName.ALGORITHM in content + assert PropName.FORMULA in content assert PropName.SECTIONS in content - assert content[PropName.ALGORITHM] in AlgoName.__dict__.values() + assert content[PropName.FORMULA] in FormulaName.__dict__.values() sections = content[PropName.SECTIONS] assert isinstance(sections, list) for section in sections: |