From d8c05b528b5a75377848d130286bc997da06a792 Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Fri, 30 Dec 2016 14:08:15 +0800 Subject: Implement plan, qpi and metric constructors - separate runner from `plan` and move `plan` to `loader` - rename `algorithm` to `formula` - use `numpy` for formulas - update sample qpi and metrics Change-Id: I3131ca79907376f2de31d6cb920bd7d0230664a6 Signed-off-by: Yujun Zhang --- tests/conftest.py | 3 +- tests/data/benchmarks/QPI/compute.yaml | 37 ++++++++----------------- tests/data/benchmarks/QPI/fake-qpi.yaml | 6 ++-- tests/data/benchmarks/metric/dhrystone.yaml | 6 ++-- tests/data/benchmarks/metric/whetstone.yaml | 9 ++++-- tests/unit/loader/plan_test.py | 43 +++++++++++++++++++++++++++++ tests/unit/loader/qpi_test.py | 6 ++-- tests/unit/runner/plan_test.py | 38 ------------------------- 8 files changed, 73 insertions(+), 75 deletions(-) create mode 100644 tests/unit/loader/plan_test.py delete mode 100644 tests/unit/runner/plan_test.py (limited to 'tests') diff --git a/tests/conftest.py b/tests/conftest.py index f1ec91fb..7acb75e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,9 +8,10 @@ ############################################################################## from os import path + import pytest -from qtip.runner.plan import Plan +from qtip.loader.plan import Plan @pytest.fixture(scope='session') diff --git a/tests/data/benchmarks/QPI/compute.yaml b/tests/data/benchmarks/QPI/compute.yaml index 5ad8a4b6..e28efaf9 100644 --- a/tests/data/benchmarks/QPI/compute.yaml +++ b/tests/data/benchmarks/QPI/compute.yaml @@ -1,44 +1,29 @@ title: compute description: sample performance index of computing -algorithm: weighted arithmetic mean +formula: weighted arithmetic mean sections: - name: Integer weight: 0.3 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dhrystone.yaml - workloads: - - single_cpu - - multi_cpu + - dhrystone.yaml - name: Float weight: 0.3 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dhrystone.yaml - workloads: - - single_cpu - - multi_cpu + - whetstone.yaml - name: Memory weight: 0.2 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: ramspeed.yaml - workloads: - - int: [add, average, copy, scale, triad] - - float: [add, average, copy, scale, triad] + - ramspeed.yaml - name: DPI weight: 0.1 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: dpi.yaml - workloads: - - bps - - pps + - dpi.yaml - name: SSL weight: 0.1 - algorithm: geometric mean + formula: geometric mean metrics: - - spec: ssl.yaml - workloads: - - aes_128_cbc: [512, 1024, 2048, 4096] - - rsa_sig: [16, 64, 256, 1024, 8192] + - ssl.yaml diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml index d75c6568..aa1097f4 100644 --- a/tests/data/benchmarks/QPI/fake-qpi.yaml +++ b/tests/data/benchmarks/QPI/fake-qpi.yaml @@ -1,9 +1,9 @@ name: Fake QPI description: a fake QPI producing random result -algorithm: weighted arithmetic mean +formula: weighted arithmetic mean sections: - name: Fake Section weight: 0.5 - algorithm: geometric mean + formula: geometric mean metrics: - - fake_metric.yaml + - fake-metric.yaml diff --git a/tests/data/benchmarks/metric/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml index b0d55ed2..220b7841 100644 --- a/tests/data/benchmarks/metric/dhrystone.yaml +++ b/tests/data/benchmarks/metric/dhrystone.yaml @@ -1,7 +1,9 @@ name: dhrystone description: > - a synthetic computing benchmark program intended to be representative of - system (integer) programming + A synthetic computing benchmark program intended to be representative of + system (integer) programming. +links: + - https://en.wikipedia.org/wiki/Dhrystone workloads: - single_cpu - multi_cpu diff --git a/tests/data/benchmarks/metric/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml index d83680c4..448c9645 100644 --- a/tests/data/benchmarks/metric/whetstone.yaml +++ b/tests/data/benchmarks/metric/whetstone.yaml @@ -1,5 +1,10 @@ -name: dhrystone -description: a synthetic benchmark for evaluating the performance of computers +name: whetstone +description: > + A synthetic benchmark for evaluating the performance of computers. + The Whetstone benchmark primarily measures the floating-point arithmetic + performance. +links: + - https://en.wikipedia.org/wiki/Whetstone_(benchmark) workloads: - single_cpu - multi_cpu diff --git a/tests/unit/loader/plan_test.py b/tests/unit/loader/plan_test.py new file mode 100644 index 00000000..6aab5e8a --- /dev/null +++ b/tests/unit/loader/plan_test.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.constant import PropName +from qtip.loader.plan import Plan, QPISpec + + +def test_init(plan): + assert plan.name == 'fake plan' + assert isinstance(plan.content, dict) + for qpi in plan.qpis: + assert isinstance(qpi, QPISpec) + + with pytest.raises(TypeError) as excinfo: + Plan() + assert '__init__() takes at least 2 arguments (1 given)' \ + in str(excinfo.value) + + +def test_list_all(benchmarks_root): + plan_list = Plan.list_all(paths=[benchmarks_root]) + assert len(list(plan_list)) is 1 + for desc in plan_list: + assert PropName.NAME in desc + assert PropName.CONTENT in desc + assert PropName.ABSPATH in desc + assert PropName.ABSPATH is not None + + +def test_content(plan): + content = plan.content + assert PropName.NAME in content + assert PropName.DESCRIPTION in content + assert PropName.CONFIG in content + assert PropName.QPIS in content diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py index c0d4b377..4b3fd4d0 100644 --- a/tests/unit/loader/qpi_test.py +++ b/tests/unit/loader/qpi_test.py @@ -9,7 +9,7 @@ import pytest -from qtip.base.constant import AlgoName, PropName +from qtip.base.constant import FormulaName, PropName from qtip.loader.qpi import QPISpec QPI_SPEC = 'compute.yaml' @@ -42,10 +42,10 @@ def test_list_all(benchmarks_root): def test_content(qpi_spec): content = qpi_spec.content assert PropName.DESCRIPTION in content - assert PropName.ALGORITHM in content + assert PropName.FORMULA in content assert PropName.SECTIONS in content - assert content[PropName.ALGORITHM] in AlgoName.__dict__.values() + assert content[PropName.FORMULA] in FormulaName.__dict__.values() sections = content[PropName.SECTIONS] assert isinstance(sections, list) for section in sections: diff --git a/tests/unit/runner/plan_test.py b/tests/unit/runner/plan_test.py deleted file mode 100644 index 7b3611d1..00000000 --- a/tests/unit/runner/plan_test.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from qtip.base.constant import PropName -from qtip.runner.plan import Plan - - -def test_init(plan): - assert plan.name == 'fake plan' - - with pytest.raises(TypeError) as excinfo: - Plan() - assert '__init__() takes at least 2 arguments (1 given)' \ - in str(excinfo.value) - - -def test_list_all(benchmarks_root): - plan_list = Plan.list_all(paths=[benchmarks_root]) - assert len(list(plan_list)) is 1 - for desc in plan_list: - assert PropName.NAME in desc - assert PropName.CONTENT in desc - assert PropName.ABSPATH in desc - assert PropName.ABSPATH is not None - - -def test_content(plan): - content = plan.content - assert PropName.NAME in content - assert PropName.DESCRIPTION in content -- cgit 1.2.3-korg