diff options
Diffstat (limited to 'tests')
18 files changed, 100 insertions, 159 deletions
diff --git a/tests/conftest.py b/tests/conftest.py index 8e2ecf7c..f1ec91fb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ from os import path import pytest +from qtip.runner.plan import Plan + @pytest.fixture(scope='session') def data_root(): @@ -19,3 +21,8 @@ def data_root(): @pytest.fixture(scope='session') def benchmarks_root(data_root): return path.join(data_root, 'benchmarks') + + +@pytest.fixture(scope='session') +def plan(benchmarks_root): + return Plan('fake-plan.yaml', [benchmarks_root]) diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml new file mode 100644 index 00000000..d75c6568 --- /dev/null +++ b/tests/data/benchmarks/QPI/fake-qpi.yaml @@ -0,0 +1,9 @@ +name: Fake QPI +description: a fake QPI producing random result +algorithm: weighted arithmetic mean +sections: +- name: Fake Section + weight: 0.5 + algorithm: geometric mean + metrics: + - fake_metric.yaml diff --git a/tests/data/benchmarks/metrics/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml index b0d55ed2..b0d55ed2 100644 --- a/tests/data/benchmarks/metrics/dhrystone.yaml +++ b/tests/data/benchmarks/metric/dhrystone.yaml diff --git a/tests/data/benchmarks/metrics/dpi.yaml b/tests/data/benchmarks/metric/dpi.yaml index fc24c8d1..fc24c8d1 100644 --- a/tests/data/benchmarks/metrics/dpi.yaml +++ b/tests/data/benchmarks/metric/dpi.yaml diff --git a/tests/data/benchmarks/metric/fake-metric.yaml b/tests/data/benchmarks/metric/fake-metric.yaml new file mode 100644 index 00000000..c5fcc5c2 --- /dev/null +++ b/tests/data/benchmarks/metric/fake-metric.yaml @@ -0,0 +1,7 @@ +name: fake compute +description: > + a fake compute performance index which generate random result +workloads: # all supported workloads + - fake_workload_1 + - fake_workload_2 + - fake_workload_3 diff --git a/tests/data/benchmarks/metrics/ramspeed.yaml b/tests/data/benchmarks/metric/ramspeed.yaml index bb7618b7..bb7618b7 100644 --- a/tests/data/benchmarks/metrics/ramspeed.yaml +++ b/tests/data/benchmarks/metric/ramspeed.yaml diff --git a/tests/data/benchmarks/metrics/ssl.yaml b/tests/data/benchmarks/metric/ssl.yaml index 21e8add7..21e8add7 100644 --- a/tests/data/benchmarks/metrics/ssl.yaml +++ b/tests/data/benchmarks/metric/ssl.yaml diff --git a/tests/data/benchmarks/metrics/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml index d83680c4..d83680c4 100644 --- a/tests/data/benchmarks/metrics/whetstone.yaml +++ b/tests/data/benchmarks/metric/whetstone.yaml diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml new file mode 100644 index 00000000..8887f66d --- /dev/null +++ b/tests/data/benchmarks/plan/fake-plan.yaml @@ -0,0 +1,10 @@ +name: fake plan +description: fake benchmark plan for demonstration and testing +config: + facility: local + engineer: local + driver: sample + collector: logfile + reporter: console +QPIs: + - fake-qpi.yaml diff --git a/tests/data/benchmarks/plans/verification.yaml b/tests/data/benchmarks/plans/verification.yaml deleted file mode 100644 index b146ee37..00000000 --- a/tests/data/benchmarks/plans/verification.yaml +++ /dev/null @@ -1,21 +0,0 @@ -title: verification -description: benchmark plan for QTIP verification -info: - facility: opnfv-ci - engineer: opnfv-bot -suites: - - QPI_spec: compute.yaml -# Uncomment next line to setup suite running condition -# condition: {} -# Uncomment the following lines setup case running configuration -# cases: -# - metric_spec: dhrystone.yaml -# config: {} -# - metric_spec: whetstone.yaml -# config: {} -# - metric_spec: ramspeed.yaml -# config: {} -# - metric_spec: dpi.yaml -# config: {} -# - metric_spec: ssl.yaml -# config: {} diff --git a/tests/unit/cli/test_module.py b/tests/unit/cli/test_module.py deleted file mode 100644 index a8900684..00000000 --- a/tests/unit/cli/test_module.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import pytest -import sys -import yaml - -from click.testing import CliRunner -from qtip.cli.entry import cli - - -class TestClass(object): - '''TODO(taseer) Remove hardcoded behaviour ''' - - @pytest.fixture() - def runner(self): - return CliRunner() - - def test(self, runner): - unit = 'perftest' - test_file = 'data/helper/' + unit + '.yaml' - path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, test_file) - - with open(path) as trial: - content = trial.read() - data = yaml.safe_load(content)['tests'] - if data is None: - print("Unit Test does not exist") - sys.exit(1) - else: - for i in range(0, len(data)): - result = runner.invoke(cli, data[i]['command']) - assert result.output == data[i]['output'] diff --git a/tests/unit/cli/test_options.py b/tests/unit/cli/test_options.py new file mode 100644 index 00000000..5129b5cc --- /dev/null +++ b/tests/unit/cli/test_options.py @@ -0,0 +1,32 @@ +############################################################### +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest +from click.testing import CliRunner + +from qtip.cli.entry import cli + + +class TestClass(object): + + @pytest.fixture() + def runner(self): + return CliRunner() + + def test_verbose(self, runner): + result = runner.invoke(cli, ['-v']) + assert result.output == '' + + def test_version(self, runner): + result = runner.invoke(cli, ['--version']) + assert '' in result.output + + def test_debug(self, runner): + result = runner.invoke(cli, ['-d']) + assert '' in result.output diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py index 5eced700..d2be0388 100644 --- a/tests/unit/loader/metric_test.py +++ b/tests/unit/loader/metric_test.py @@ -9,8 +9,8 @@ import pytest -from qtip.base.benchmark import Property -from qtip.spec.metric import MetricSpec +from qtip.base.constant import PropName +from qtip.loader.metric import MetricSpec @pytest.fixture(scope='module') @@ -29,17 +29,17 @@ def init_test(metric_spec): def list_all_test(): metric_list = MetricSpec.list_all() - assert len(list(metric_list)) is 1 + assert len(list(metric_list)) is 6 for desc in metric_list: - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc - assert Property.ABSPATH is not None + assert PropName.NAME in desc + assert PropName.DESCRIPTION in desc + assert PropName.ABSPATH in desc + assert PropName.ABSPATH is not None def content_test(metric): - content = metric.content() - assert Property.NAME in content - assert Property.DESCRIPTION in content - assert Property.WORKLOADS in content - assert isinstance(content[Property.WORKLOADS], list) + content = metric.content + assert PropName.NAME in content + assert PropName.DESCRIPTION in content + assert PropName.WORKLOADS in content + assert isinstance(content[PropName.WORKLOADS], list) diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py index bfa1f580..c0d4b377 100644 --- a/tests/unit/loader/qpi_test.py +++ b/tests/unit/loader/qpi_test.py @@ -9,8 +9,8 @@ import pytest -from qtip.base.benchmark import Algorithm, Property -from qtip.spec.qpi import QPISpec +from qtip.base.constant import AlgoName, PropName +from qtip.loader.qpi import QPISpec QPI_SPEC = 'compute.yaml' @@ -31,22 +31,22 @@ def test_init(qpi_spec): def test_list_all(benchmarks_root): qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root]) - assert len(list(qpi_spec_list)) is 1 + assert len(list(qpi_spec_list)) is 2 for item in qpi_spec_list: - assert Property.NAME in item - assert Property.CONTENT in item - assert Property.ABSPATH in item - assert Property.ABSPATH is not None + assert PropName.NAME in item + assert PropName.CONTENT in item + assert PropName.ABSPATH in item + assert PropName.ABSPATH is not None def test_content(qpi_spec): - content = qpi_spec.content() - assert Property.DESCRIPTION in content - assert Property.ALGORITHM in content - assert Property.SECTIONS in content + content = qpi_spec.content + assert PropName.DESCRIPTION in content + assert PropName.ALGORITHM in content + assert PropName.SECTIONS in content - assert content[Property.ALGORITHM] in Algorithm.__dict__.values() - sections = content[Property.SECTIONS] + assert content[PropName.ALGORITHM] in AlgoName.__dict__.values() + sections = content[PropName.SECTIONS] assert isinstance(sections, list) for section in sections: - assert Property.NAME in section + assert PropName.NAME in section diff --git a/tests/unit/runner/case_test.py b/tests/unit/runner/case_test.py deleted file mode 100644 index 59a54a84..00000000 --- a/tests/unit/runner/case_test.py +++ /dev/null @@ -1,15 +0,0 @@ -############################################################### -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from qtip.spec.metric import MetricSpec - - -def init_test(case): - assert isinstance(case.metric_spec, MetricSpec) - assert isinstance(case.config, dict) diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py deleted file mode 100644 index 6d14f7ae..00000000 --- a/tests/unit/runner/conftest.py +++ /dev/null @@ -1,30 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from qtip.base.benchmark import Property -from qtip.runner.case import Case -from qtip.runner.plan import Plan -from qtip.runner.suite import Suite - - -@pytest.fixture(scope='module') -def plan(benchmarks_root): - return Plan('verification.yaml', paths=[benchmarks_root]) - - -@pytest.fixture(scope='module') -def suite(plan): - return Suite(plan[Property.SUITES][0]) - - -@pytest.fixture(scope='module') -def case(suite): - return Case(suite[Property.CASES][0]) diff --git a/tests/unit/runner/plan_test.py b/tests/unit/runner/plan_test.py index d783e5e9..7b3611d1 100644 --- a/tests/unit/runner/plan_test.py +++ b/tests/unit/runner/plan_test.py @@ -9,12 +9,12 @@ import pytest -from qtip.base.benchmark import Property +from qtip.base.constant import PropName from qtip.runner.plan import Plan def test_init(plan): - assert plan.name == 'verification' + assert plan.name == 'fake plan' with pytest.raises(TypeError) as excinfo: Plan() @@ -26,13 +26,13 @@ def test_list_all(benchmarks_root): plan_list = Plan.list_all(paths=[benchmarks_root]) assert len(list(plan_list)) is 1 for desc in plan_list: - assert Property.NAME in desc - assert Property.CONTENT in desc - assert Property.ABSPATH in desc - assert Property.ABSPATH is not None + assert PropName.NAME in desc + assert PropName.CONTENT in desc + assert PropName.ABSPATH in desc + assert PropName.ABSPATH is not None def test_content(plan): - content = plan.content() - assert Property.TITLE in content - assert Property.DESCRIPTION in content + content = plan.content + assert PropName.NAME in content + assert PropName.DESCRIPTION in content diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py deleted file mode 100644 index 7dad8f62..00000000 --- a/tests/unit/runner/suite_test.py +++ /dev/null @@ -1,27 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from qtip.runner.suite import Suite -from qtip.runner.case import Case -from qtip.spec.qpi import QPISpec - - -def init_test(suite): - assert isinstance(suite.qpi, QPISpec) - assert isinstance(suite.condition, dict) - assert isinstance(suite.cases, list) - for case in suite.cases: - assert isinstance(case, Case) - - with pytest.raises(TypeError) as excinfo: - Suite() - assert '__init__() takes exactly 2 arguments (1 given)' \ - in str(excinfo.value) |