From 929bcdf94d14062e042d9f9451c28315a18e808d Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Wed, 21 Dec 2016 00:19:46 +0800 Subject: Implment https://wiki.opnfv.org/display/qtip/Design Note that some obsolete test cases are marked expected failure, will be deprecated after architecture evolution. JIRA: QTIP-148 Change-Id: I52bc9391569d516e298d9e659517161b4dce794a Signed-off-by: Yujun Zhang --- tests/unit/loader/__init__.py | 0 tests/unit/loader/metric_test.py | 45 ++++++++++++++++++++++++++++++ tests/unit/loader/qpi_test.py | 52 +++++++++++++++++++++++++++++++++++ tests/unit/runner/case_test.py | 15 ++++++++++ tests/unit/runner/conftest.py | 30 ++++++++++++++++++++ tests/unit/runner/perftest_test.py | 48 -------------------------------- tests/unit/runner/plan_test.py | 38 +++++++++++++++++++++++++ tests/unit/runner/suite_test.py | 45 ++++++++---------------------- tests/unit/runner/testplan_test.py | 48 -------------------------------- tests/unit/utils/args_handler_test.py | 2 +- tests/unit/utils/cli_test.py | 1 + tests/unit/utils/create_zones_test.py | 2 +- tests/unit/utils/env_setup_test.py | 2 +- 13 files changed, 196 insertions(+), 132 deletions(-) create mode 100644 tests/unit/loader/__init__.py create mode 100644 tests/unit/loader/metric_test.py create mode 100644 tests/unit/loader/qpi_test.py create mode 100644 tests/unit/runner/case_test.py create mode 100644 tests/unit/runner/conftest.py delete mode 100644 tests/unit/runner/perftest_test.py create mode 100644 tests/unit/runner/plan_test.py delete mode 100644 tests/unit/runner/testplan_test.py (limited to 'tests/unit') diff --git a/tests/unit/loader/__init__.py b/tests/unit/loader/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py new file mode 100644 index 00000000..5eced700 --- /dev/null +++ b/tests/unit/loader/metric_test.py @@ -0,0 +1,45 @@ +############################################################### +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.benchmark import Property +from qtip.spec.metric import MetricSpec + + +@pytest.fixture(scope='module') +def metric_spec(benchmarks_root): + return MetricSpec('dhrystone.yaml', paths=[benchmarks_root]) + + +def init_test(metric_spec): + assert metric_spec.name == 'dhrystone' + + with pytest.raises(TypeError) as excinfo: + MetricSpec() + assert '__init__() takes at least 2 arguments (1 given)' \ + in str(excinfo.value) + + +def list_all_test(): + metric_list = MetricSpec.list_all() + assert len(list(metric_list)) is 1 + for desc in metric_list: + assert Property.NAME in desc + assert Property.DESCRIPTION in desc + assert Property.ABSPATH in desc + assert Property.ABSPATH is not None + + +def content_test(metric): + content = metric.content() + assert Property.NAME in content + assert Property.DESCRIPTION in content + assert Property.WORKLOADS in content + assert isinstance(content[Property.WORKLOADS], list) diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py new file mode 100644 index 00000000..bfa1f580 --- /dev/null +++ b/tests/unit/loader/qpi_test.py @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.benchmark import Algorithm, Property +from qtip.spec.qpi import QPISpec + +QPI_SPEC = 'compute.yaml' + + +@pytest.fixture() +def qpi_spec(benchmarks_root): + return QPISpec('compute.yaml', paths=[benchmarks_root]) + + +def test_init(qpi_spec): + assert qpi_spec.name == 'compute' + + with pytest.raises(TypeError) as excinfo: + QPISpec() + assert '__init__() takes at least 2 arguments (1 given)' \ + in str(excinfo.value) + + +def test_list_all(benchmarks_root): + qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root]) + assert len(list(qpi_spec_list)) is 1 + for item in qpi_spec_list: + assert Property.NAME in item + assert Property.CONTENT in item + assert Property.ABSPATH in item + assert Property.ABSPATH is not None + + +def test_content(qpi_spec): + content = qpi_spec.content() + assert Property.DESCRIPTION in content + assert Property.ALGORITHM in content + assert Property.SECTIONS in content + + assert content[Property.ALGORITHM] in Algorithm.__dict__.values() + sections = content[Property.SECTIONS] + assert isinstance(sections, list) + for section in sections: + assert Property.NAME in section diff --git a/tests/unit/runner/case_test.py b/tests/unit/runner/case_test.py new file mode 100644 index 00000000..59a54a84 --- /dev/null +++ b/tests/unit/runner/case_test.py @@ -0,0 +1,15 @@ +############################################################### +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.spec.metric import MetricSpec + + +def init_test(case): + assert isinstance(case.metric_spec, MetricSpec) + assert isinstance(case.config, dict) diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py new file mode 100644 index 00000000..6d14f7ae --- /dev/null +++ b/tests/unit/runner/conftest.py @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.benchmark import Property +from qtip.runner.case import Case +from qtip.runner.plan import Plan +from qtip.runner.suite import Suite + + +@pytest.fixture(scope='module') +def plan(benchmarks_root): + return Plan('verification.yaml', paths=[benchmarks_root]) + + +@pytest.fixture(scope='module') +def suite(plan): + return Suite(plan[Property.SUITES][0]) + + +@pytest.fixture(scope='module') +def case(suite): + return Case(suite[Property.CASES][0]) diff --git a/tests/unit/runner/perftest_test.py b/tests/unit/runner/perftest_test.py deleted file mode 100644 index 2b400ac8..00000000 --- a/tests/unit/runner/perftest_test.py +++ /dev/null @@ -1,48 +0,0 @@ -############################################################### -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from os import path -import pytest - -from qtip.runner.perftest import PerfTest -from qtip.runner.benchmark import Property - - -class TestPerfTestClass: - def test_attr(self): - assert len(PerfTest._paths) is 1 - - -class TestPerfTest: - PerfTest._paths = [path.join(path.dirname(__file__), path.pardir, - path.pardir, 'data', 'perftest')] - - def test_init(self): - perftest = PerfTest('test-a') - assert perftest.name == 'test-a' - - with pytest.raises(TypeError) as excinfo: - PerfTest() - assert '__init__() takes exactly 2 arguments (1 given)' \ - in str(excinfo.value) - - def test_list(self): - perftest_list = PerfTest.list_all() - assert len(list(perftest_list)) is 1 - for desc in perftest_list: - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc - assert Property.ABSPATH is not None - - def test_describe(self): - desc = PerfTest('test-a').describe() - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc diff --git a/tests/unit/runner/plan_test.py b/tests/unit/runner/plan_test.py new file mode 100644 index 00000000..d783e5e9 --- /dev/null +++ b/tests/unit/runner/plan_test.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.benchmark import Property +from qtip.runner.plan import Plan + + +def test_init(plan): + assert plan.name == 'verification' + + with pytest.raises(TypeError) as excinfo: + Plan() + assert '__init__() takes at least 2 arguments (1 given)' \ + in str(excinfo.value) + + +def test_list_all(benchmarks_root): + plan_list = Plan.list_all(paths=[benchmarks_root]) + assert len(list(plan_list)) is 1 + for desc in plan_list: + assert Property.NAME in desc + assert Property.CONTENT in desc + assert Property.ABSPATH in desc + assert Property.ABSPATH is not None + + +def test_content(plan): + content = plan.content() + assert Property.TITLE in content + assert Property.DESCRIPTION in content diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py index acfed82c..7dad8f62 100644 --- a/tests/unit/runner/suite_test.py +++ b/tests/unit/runner/suite_test.py @@ -7,42 +7,21 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from os import path import pytest from qtip.runner.suite import Suite -from qtip.runner.benchmark import Property +from qtip.runner.case import Case +from qtip.spec.qpi import QPISpec -class TestSuiteClass: - def test_attr(self): - assert len(Suite._paths) is 1 +def init_test(suite): + assert isinstance(suite.qpi, QPISpec) + assert isinstance(suite.condition, dict) + assert isinstance(suite.cases, list) + for case in suite.cases: + assert isinstance(case, Case) - -class TestSuite: - Suite._paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, - 'data', 'suite')] - - def test_init(self): - suite = Suite('suite-1') - assert suite.name == 'suite-1' - - with pytest.raises(TypeError) as excinfo: - Suite() - assert '__init__() takes exactly 2 arguments (1 given)' \ - in str(excinfo.value) - - def test_list(self): - suite_list = Suite.list_all() - assert len(list(suite_list)) is 3 - for suite_desc in suite_list: - assert Property.NAME in suite_desc - assert Property.DESCRIPTION in suite_desc - assert Property.ABSPATH in suite_desc - assert Property.ABSPATH is not None - - def test_describe(self): - desc = Suite('suite-a').describe() - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc + with pytest.raises(TypeError) as excinfo: + Suite() + assert '__init__() takes exactly 2 arguments (1 given)' \ + in str(excinfo.value) diff --git a/tests/unit/runner/testplan_test.py b/tests/unit/runner/testplan_test.py deleted file mode 100644 index 7e42f557..00000000 --- a/tests/unit/runner/testplan_test.py +++ /dev/null @@ -1,48 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from os import path -import pytest - -from qtip.runner.testplan import TestPlan -from qtip.runner.benchmark import Property - - -class TestTestPlanClass: - def test_attr(self): - assert len(TestPlan._paths) is 1 - - -class TestTestPlan: - TestPlan._paths = [path.join(path.dirname(__file__), path.pardir, - path.pardir, 'data', 'testplan')] - - def test_init(self): - plan = TestPlan('plan-a') - assert plan.name == 'plan-a' - - with pytest.raises(TypeError) as excinfo: - TestPlan() - assert '__init__() takes exactly 2 arguments (1 given)' \ - in str(excinfo.value) - - def test_list(self): - plan_list = TestPlan.list_all() - assert len(list(plan_list)) is 5 - for desc in plan_list: - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc - assert Property.ABSPATH is not None - - def test_describe(self): - desc = TestPlan('plan-a').describe() - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc diff --git a/tests/unit/utils/args_handler_test.py b/tests/unit/utils/args_handler_test.py index 5cbe766e..dceca1f5 100644 --- a/tests/unit/utils/args_handler_test.py +++ b/tests/unit/utils/args_handler_test.py @@ -11,8 +11,8 @@ import mock import qtip.utils.args_handler +@pytest.mark.xfail(reason="to be fixed") class TestClass: - @pytest.mark.skip("(yujunz) test fails") @pytest.mark.parametrize("test_input, expected", [ (['fuel', '/home', 'benchmarks/testplan/default/network/iperf_bm.yaml'], ['fuel', '/home', "iperf", diff --git a/tests/unit/utils/cli_test.py b/tests/unit/utils/cli_test.py index 13bb857b..0f3e4158 100644 --- a/tests/unit/utils/cli_test.py +++ b/tests/unit/utils/cli_test.py @@ -5,6 +5,7 @@ from qtip.utils.cli import Cli from os.path import expanduser +@pytest.mark.skip("TODO(yujunz) recover test after refactoring") class TestClass: @pytest.mark.parametrize("test_input, expected", [ (['-l', diff --git a/tests/unit/utils/create_zones_test.py b/tests/unit/utils/create_zones_test.py index 4a86b5cb..dcfff5ec 100644 --- a/tests/unit/utils/create_zones_test.py +++ b/tests/unit/utils/create_zones_test.py @@ -50,8 +50,8 @@ class NovaMock(MagicMock): aggregates = AggMock() +@pytest.mark.xfail(reason="unstable result") class TestClass: - @pytest.mark.skip("(yujunz) unstable result") @pytest.mark.parametrize("test_input, expected", [ (['compute1', 'compute2'], ['create:compute1:compute1', diff --git a/tests/unit/utils/env_setup_test.py b/tests/unit/utils/env_setup_test.py index c40801cc..dea48190 100644 --- a/tests/unit/utils/env_setup_test.py +++ b/tests/unit/utils/env_setup_test.py @@ -78,7 +78,7 @@ class TestClass: result = filecmp.cmp(get_output("hosts"), "config/hosts") assert result - @pytest.mark.skip("(yujunz) to be fixed") + @pytest.mark.skip("(yujunz) test hung") def test_ping(self, capfd): test_class = Env_setup() mock_ips = mock.Mock(return_value=["127.0.0.1", "10.20.0.29"]) -- cgit 1.2.3-korg