diff options
32 files changed, 481 insertions, 232 deletions
diff --git a/tests/data/perftest/iperf b/qtip/base/__init__.py index e69de29b..e69de29b 100644 --- a/tests/data/perftest/iperf +++ b/qtip/base/__init__.py diff --git a/qtip/base/benchmark.py b/qtip/base/benchmark.py new file mode 100644 index 00000000..b38e6016 --- /dev/null +++ b/qtip/base/benchmark.py @@ -0,0 +1,95 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from itertools import chain +from os import listdir +from os import path +import yaml + + +class Property(object): + # list + NAME = 'name' + CONTENT = 'content' + ABSPATH = 'abspath' + # content + TITLE = 'title' + DESCRIPTION = 'description' + # spec + ALGORITHM = 'algorithm' + SECTIONS = 'sections' + WEIGHT = 'weight' + METRICS = 'metrics' + SPEC = 'spec' + WORKLOADS = 'workloads' + # plan + INFO = 'info' + FACILITY = 'facility' + ENGINEER = 'engineer' + SUITES = 'suites' + # suite + QPI_SPEC = 'QPI_spec' + CONDITION = 'condition' + CASES = 'cases' + # case + METRIC_SPEC = 'metric_spec' + CONFIG = 'config' + + +class QtipError(Exception): + pass + + +class Algorithm(object): + ARITHMETIC_MEAN = 'arithmetic mean' + WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean' + GEOMETRIC_MEAN = 'geometric mean' + WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean' + + +ROOT_DIR = 'benchmarks' + + +class Benchmark(object): + """Abstract class of QTIP benchmarks""" + DEFAULT_DIR = '.' + _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, + ROOT_DIR)] + + def __init__(self, name, paths=None): + self._file = name + self._abspath = self._find(name, paths) + self.name = path.splitext(name)[0] + + def _find(self, name, paths): + """find a benchmark in searching paths""" + paths = self._paths if paths is None else paths + name = path.join(self.DEFAULT_DIR, name) + for p in paths: + abspath = path.join(p, name) + if path.exists(abspath): + return abspath + raise QtipError("'{}' not found in paths: {}".format(name, paths)) + + @classmethod + def list_all(cls, paths=None): + """list all available benchmarks""" + paths = cls._paths if paths is None else paths + names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR)) + for p in paths]) + for name in names: + item = cls(name, paths=paths) + yield { + Property.NAME: name, + Property.ABSPATH: item._abspath, + Property.CONTENT: item.content()} + + def content(self): + """description of benchmark""" + return yaml.safe_load(file(self._abspath)) diff --git a/qtip/cli/commands/cmd_testplan.py b/qtip/cli/commands/cmd_plan.py index 200e4665..01bf8251 100644 --- a/qtip/cli/commands/cmd_testplan.py +++ b/qtip/cli/commands/cmd_plan.py @@ -8,10 +8,8 @@ ############################################################################## import click -import sys from prettytable import PrettyTable - -from qtip.runner.testplan import TestPlan +from qtip.runner.plan import Plan @click.group() @@ -20,30 +18,26 @@ def cli(): @cli.group() -def testplan(): +def plan_cmd(): pass -@testplan.command('list', help='List the different TestPlans.') -def list(): - testplans = TestPlan.list_all() +@plan_cmd.command('list', help='List the different TestPlans.') +def list_all(): + plans = Plan.list_all() table = PrettyTable(["Testplans"]) table.align = 'l' - for testplan in testplans: - table.add_row([testplan['name']]) + for plan in plans: + table.add_row([plan['name']]) click.echo(table) -@testplan.command('show', help='Show details of specified TestPlan.') +@plan_cmd.command('show', help='Show details of specified TestPlan.') @click.argument('name') def show(name): - plan = TestPlan(name) - desc = plan.describe() - if desc['abspath'] is None: - click.echo("Wrong TestPlan specified") - sys.exit(1) - else: - table = PrettyTable(["Name", "Description"]) - table.align = 'l' - table.add_row([desc['name'], desc['description']]) - click.echo(table) + plan = Plan(name) + results = plan.content() + table = PrettyTable(["Name", "Description"]) + table.align = 'l' + table.add_row([results['name'], results['description']]) + click.echo(table) diff --git a/qtip/runner/benchmark.py b/qtip/runner/benchmark.py deleted file mode 100644 index 58e5a8d9..00000000 --- a/qtip/runner/benchmark.py +++ /dev/null @@ -1,52 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from itertools import chain -from os import listdir -from os import path - - -class Property: - NAME = 'name' - DESCRIPTION = 'description' - ABSPATH = 'abspath' - - -class Benchmark(object): - """Abstract class of QTIP benchmarks""" - - _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, - 'benchmarks')] - - def __init__(self, name): - self.name = name - self._abspath = self._find(name) - - def _find(self, name): - """find a benchmark in searching paths""" - for p in self._paths: - abspath = path.join(p, name) - if path.exists(abspath): - return abspath - return None - - @classmethod - def list_all(cls): - """list all available benchmarks""" - names = chain.from_iterable([listdir(p) for p in cls._paths]) - return [cls(name).describe() for name in names] - - def describe(self): - """description of benchmark""" - # TODO(yujunz) read description from benchmark content - return { - Property.NAME: self.name, - Property.DESCRIPTION: 'QTIP benchmark', - Property.ABSPATH: self._abspath - } diff --git a/qtip/runner/perftest.py b/qtip/runner/case.py index a9b54716..eb3febc2 100644 --- a/qtip/runner/perftest.py +++ b/qtip/runner/case.py @@ -7,15 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from os import path +from qtip.base.benchmark import Property +from qtip.spec.metric import MetricSpec -from benchmark import Benchmark - -class PerfTest(Benchmark): - """WIP(yujunz): - a perftest is the driver of external performance test tools - It is usually referred in a suite to collect performance metric""" - - # paths to search for perftest - _paths = [path.join(p, 'perftest') for p in Benchmark._paths] +class Case(object): + def __init__(self, spec, paths=None): + self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths) + self.config = spec[Property.CONFIG] diff --git a/qtip/runner/plan.py b/qtip/runner/plan.py new file mode 100644 index 00000000..265ad8d7 --- /dev/null +++ b/qtip/runner/plan.py @@ -0,0 +1,26 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.base.benchmark import Benchmark, Property +from qtip.runner.suite import Suite + + +class Plan(Benchmark): + """ + a benchmark plan is consist of basic information and several suites""" + + DEFAULT_DIR = 'plans' + + def __init__(self, name, paths=None): + super(Plan, self).__init__(name, paths=paths) + content = self.content() + + self.info = content[Property.INFO] + self.suites = [Suite(suite, paths=paths) + for suite in content[Property.SUITES]] diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py index 0086a20e..55033d2b 100644 --- a/qtip/runner/suite.py +++ b/qtip/runner/suite.py @@ -7,16 +7,16 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from os import path +from qtip.base.benchmark import Property +from qtip.spec.qpi import QPISpec +from qtip.runner.case import Case -from benchmark import Benchmark - -class Suite(Benchmark): - """WIP(yujunz): - a suite is consist of one or several perf tests and produces one QPI. - It must be executed as part of testplan - """ - - # paths to search for suites - _paths = [path.join(p, 'suite') for p in Benchmark._paths] +class Suite(object): + """a suite of benchmark cases under specified condition""" + def __init__(self, spec, paths=None): + self._paths = paths + self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths) + self.condition = spec.get(Property.CONDITION, {}) + self.cases = [Case(case_spec, paths) + for case_spec in spec.get(Property.CASES, [])] diff --git a/tests/data/suite/suite-1 b/qtip/spec/__init__.py index e69de29b..e69de29b 100644 --- a/tests/data/suite/suite-1 +++ b/qtip/spec/__init__.py diff --git a/qtip/spec/metric.py b/qtip/spec/metric.py new file mode 100644 index 00000000..e9c70547 --- /dev/null +++ b/qtip/spec/metric.py @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.base.benchmark import Benchmark + + +class MetricSpec(Benchmark): + """metrics in QTIP are categorized by performance test tools, such as + dhrystone, whetstone and etc""" + DEFAULT_DIR = 'metrics' diff --git a/qtip/spec/qpi.py b/qtip/spec/qpi.py new file mode 100644 index 00000000..b7d7aa02 --- /dev/null +++ b/qtip/spec/qpi.py @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.base.benchmark import Benchmark + + +class QPISpec(Benchmark): + """ + a QPI specification defines how to calculate a performance index from + collected metrics. + """ + DEFAULT_DIR = 'QPI' diff --git a/qtip/runner/testplan.py b/tests/conftest.py index f48a7147..8e2ecf7c 100644 --- a/qtip/runner/testplan.py +++ b/tests/conftest.py @@ -8,14 +8,14 @@ ############################################################################## from os import path +import pytest -from benchmark import Benchmark +@pytest.fixture(scope='session') +def data_root(): + return path.join(path.dirname(__file__), 'data') -class TestPlan(Benchmark): - """WIP(yujunz): - a test plan is consist of test condition and several suites which can be - executed by user""" - # paths to search for suites - _paths = [path.join(p, 'testplan') for p in Benchmark._paths] +@pytest.fixture(scope='session') +def benchmarks_root(data_root): + return path.join(data_root, 'benchmarks') diff --git a/tests/data/benchmarks/QPI/compute.yaml b/tests/data/benchmarks/QPI/compute.yaml new file mode 100644 index 00000000..5ad8a4b6 --- /dev/null +++ b/tests/data/benchmarks/QPI/compute.yaml @@ -0,0 +1,44 @@ +title: compute +description: sample performance index of computing +algorithm: weighted arithmetic mean +sections: +- name: Integer + weight: 0.3 + algorithm: geometric mean + metrics: + - spec: dhrystone.yaml + workloads: + - single_cpu + - multi_cpu +- name: Float + weight: 0.3 + algorithm: geometric mean + metrics: + - spec: dhrystone.yaml + workloads: + - single_cpu + - multi_cpu +- name: Memory + weight: 0.2 + algorithm: geometric mean + metrics: + - spec: ramspeed.yaml + workloads: + - int: [add, average, copy, scale, triad] + - float: [add, average, copy, scale, triad] +- name: DPI + weight: 0.1 + algorithm: geometric mean + metrics: + - spec: dpi.yaml + workloads: + - bps + - pps +- name: SSL + weight: 0.1 + algorithm: geometric mean + metrics: + - spec: ssl.yaml + workloads: + - aes_128_cbc: [512, 1024, 2048, 4096] + - rsa_sig: [16, 64, 256, 1024, 8192] diff --git a/tests/data/benchmarks/metrics/dhrystone.yaml b/tests/data/benchmarks/metrics/dhrystone.yaml new file mode 100644 index 00000000..b0d55ed2 --- /dev/null +++ b/tests/data/benchmarks/metrics/dhrystone.yaml @@ -0,0 +1,7 @@ +name: dhrystone +description: > + a synthetic computing benchmark program intended to be representative of + system (integer) programming +workloads: + - single_cpu + - multi_cpu diff --git a/tests/data/benchmarks/metrics/dpi.yaml b/tests/data/benchmarks/metrics/dpi.yaml new file mode 100644 index 00000000..fc24c8d1 --- /dev/null +++ b/tests/data/benchmarks/metrics/dpi.yaml @@ -0,0 +1,5 @@ +name: dpi +description: deep packet inspection +workloads: + - bps + - pps diff --git a/tests/data/benchmarks/metrics/ramspeed.yaml b/tests/data/benchmarks/metrics/ramspeed.yaml new file mode 100644 index 00000000..bb7618b7 --- /dev/null +++ b/tests/data/benchmarks/metrics/ramspeed.yaml @@ -0,0 +1,5 @@ +name: ramspeed +description: a memory performance benchmarking +workloads: + - int: [add, average, copy, scale, triad] + - float: [add, average, copy, scale, triad] diff --git a/tests/data/benchmarks/metrics/ssl.yaml b/tests/data/benchmarks/metrics/ssl.yaml new file mode 100644 index 00000000..21e8add7 --- /dev/null +++ b/tests/data/benchmarks/metrics/ssl.yaml @@ -0,0 +1,5 @@ +name: ssl +description: a Secure Sockets Layer performance benchmarking +workloads: + - aes_128_cbc: [512, 1024, 2048, 4096] + - rsa_sig: [16, 64, 256, 1024, 8192] diff --git a/tests/data/benchmarks/metrics/whetstone.yaml b/tests/data/benchmarks/metrics/whetstone.yaml new file mode 100644 index 00000000..d83680c4 --- /dev/null +++ b/tests/data/benchmarks/metrics/whetstone.yaml @@ -0,0 +1,5 @@ +name: dhrystone +description: a synthetic benchmark for evaluating the performance of computers +workloads: + - single_cpu + - multi_cpu diff --git a/tests/data/benchmarks/plans/verification.yaml b/tests/data/benchmarks/plans/verification.yaml new file mode 100644 index 00000000..b146ee37 --- /dev/null +++ b/tests/data/benchmarks/plans/verification.yaml @@ -0,0 +1,21 @@ +title: verification +description: benchmark plan for QTIP verification +info: + facility: opnfv-ci + engineer: opnfv-bot +suites: + - QPI_spec: compute.yaml +# Uncomment next line to setup suite running condition +# condition: {} +# Uncomment the following lines setup case running configuration +# cases: +# - metric_spec: dhrystone.yaml +# config: {} +# - metric_spec: whetstone.yaml +# config: {} +# - metric_spec: ramspeed.yaml +# config: {} +# - metric_spec: dpi.yaml +# config: {} +# - metric_spec: ssl.yaml +# config: {} diff --git a/tests/data/suite/suite-3 b/tests/data/suite/suite-3 deleted file mode 100644 index e69de29b..00000000 --- a/tests/data/suite/suite-3 +++ /dev/null diff --git a/tests/data/suite/suite-2 b/tests/unit/loader/__init__.py index e69de29b..e69de29b 100644 --- a/tests/data/suite/suite-2 +++ b/tests/unit/loader/__init__.py diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py new file mode 100644 index 00000000..5eced700 --- /dev/null +++ b/tests/unit/loader/metric_test.py @@ -0,0 +1,45 @@ +############################################################### +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.benchmark import Property +from qtip.spec.metric import MetricSpec + + +@pytest.fixture(scope='module') +def metric_spec(benchmarks_root): + return MetricSpec('dhrystone.yaml', paths=[benchmarks_root]) + + +def init_test(metric_spec): + assert metric_spec.name == 'dhrystone' + + with pytest.raises(TypeError) as excinfo: + MetricSpec() + assert '__init__() takes at least 2 arguments (1 given)' \ + in str(excinfo.value) + + +def list_all_test(): + metric_list = MetricSpec.list_all() + assert len(list(metric_list)) is 1 + for desc in metric_list: + assert Property.NAME in desc + assert Property.DESCRIPTION in desc + assert Property.ABSPATH in desc + assert Property.ABSPATH is not None + + +def content_test(metric): + content = metric.content() + assert Property.NAME in content + assert Property.DESCRIPTION in content + assert Property.WORKLOADS in content + assert isinstance(content[Property.WORKLOADS], list) diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py new file mode 100644 index 00000000..bfa1f580 --- /dev/null +++ b/tests/unit/loader/qpi_test.py @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.benchmark import Algorithm, Property +from qtip.spec.qpi import QPISpec + +QPI_SPEC = 'compute.yaml' + + +@pytest.fixture() +def qpi_spec(benchmarks_root): + return QPISpec('compute.yaml', paths=[benchmarks_root]) + + +def test_init(qpi_spec): + assert qpi_spec.name == 'compute' + + with pytest.raises(TypeError) as excinfo: + QPISpec() + assert '__init__() takes at least 2 arguments (1 given)' \ + in str(excinfo.value) + + +def test_list_all(benchmarks_root): + qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root]) + assert len(list(qpi_spec_list)) is 1 + for item in qpi_spec_list: + assert Property.NAME in item + assert Property.CONTENT in item + assert Property.ABSPATH in item + assert Property.ABSPATH is not None + + +def test_content(qpi_spec): + content = qpi_spec.content() + assert Property.DESCRIPTION in content + assert Property.ALGORITHM in content + assert Property.SECTIONS in content + + assert content[Property.ALGORITHM] in Algorithm.__dict__.values() + sections = content[Property.SECTIONS] + assert isinstance(sections, list) + for section in sections: + assert Property.NAME in section diff --git a/tests/unit/runner/case_test.py b/tests/unit/runner/case_test.py new file mode 100644 index 00000000..59a54a84 --- /dev/null +++ b/tests/unit/runner/case_test.py @@ -0,0 +1,15 @@ +############################################################### +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.spec.metric import MetricSpec + + +def init_test(case): + assert isinstance(case.metric_spec, MetricSpec) + assert isinstance(case.config, dict) diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py new file mode 100644 index 00000000..6d14f7ae --- /dev/null +++ b/tests/unit/runner/conftest.py @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.benchmark import Property +from qtip.runner.case import Case +from qtip.runner.plan import Plan +from qtip.runner.suite import Suite + + +@pytest.fixture(scope='module') +def plan(benchmarks_root): + return Plan('verification.yaml', paths=[benchmarks_root]) + + +@pytest.fixture(scope='module') +def suite(plan): + return Suite(plan[Property.SUITES][0]) + + +@pytest.fixture(scope='module') +def case(suite): + return Case(suite[Property.CASES][0]) diff --git a/tests/unit/runner/perftest_test.py b/tests/unit/runner/perftest_test.py deleted file mode 100644 index 2b400ac8..00000000 --- a/tests/unit/runner/perftest_test.py +++ /dev/null @@ -1,48 +0,0 @@ -############################################################### -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from os import path -import pytest - -from qtip.runner.perftest import PerfTest -from qtip.runner.benchmark import Property - - -class TestPerfTestClass: - def test_attr(self): - assert len(PerfTest._paths) is 1 - - -class TestPerfTest: - PerfTest._paths = [path.join(path.dirname(__file__), path.pardir, - path.pardir, 'data', 'perftest')] - - def test_init(self): - perftest = PerfTest('test-a') - assert perftest.name == 'test-a' - - with pytest.raises(TypeError) as excinfo: - PerfTest() - assert '__init__() takes exactly 2 arguments (1 given)' \ - in str(excinfo.value) - - def test_list(self): - perftest_list = PerfTest.list_all() - assert len(list(perftest_list)) is 1 - for desc in perftest_list: - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc - assert Property.ABSPATH is not None - - def test_describe(self): - desc = PerfTest('test-a').describe() - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc diff --git a/tests/unit/runner/plan_test.py b/tests/unit/runner/plan_test.py new file mode 100644 index 00000000..d783e5e9 --- /dev/null +++ b/tests/unit/runner/plan_test.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.base.benchmark import Property +from qtip.runner.plan import Plan + + +def test_init(plan): + assert plan.name == 'verification' + + with pytest.raises(TypeError) as excinfo: + Plan() + assert '__init__() takes at least 2 arguments (1 given)' \ + in str(excinfo.value) + + +def test_list_all(benchmarks_root): + plan_list = Plan.list_all(paths=[benchmarks_root]) + assert len(list(plan_list)) is 1 + for desc in plan_list: + assert Property.NAME in desc + assert Property.CONTENT in desc + assert Property.ABSPATH in desc + assert Property.ABSPATH is not None + + +def test_content(plan): + content = plan.content() + assert Property.TITLE in content + assert Property.DESCRIPTION in content diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py index acfed82c..7dad8f62 100644 --- a/tests/unit/runner/suite_test.py +++ b/tests/unit/runner/suite_test.py @@ -7,42 +7,21 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from os import path import pytest from qtip.runner.suite import Suite -from qtip.runner.benchmark import Property +from qtip.runner.case import Case +from qtip.spec.qpi import QPISpec -class TestSuiteClass: - def test_attr(self): - assert len(Suite._paths) is 1 +def init_test(suite): + assert isinstance(suite.qpi, QPISpec) + assert isinstance(suite.condition, dict) + assert isinstance(suite.cases, list) + for case in suite.cases: + assert isinstance(case, Case) - -class TestSuite: - Suite._paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, - 'data', 'suite')] - - def test_init(self): - suite = Suite('suite-1') - assert suite.name == 'suite-1' - - with pytest.raises(TypeError) as excinfo: - Suite() - assert '__init__() takes exactly 2 arguments (1 given)' \ - in str(excinfo.value) - - def test_list(self): - suite_list = Suite.list_all() - assert len(list(suite_list)) is 3 - for suite_desc in suite_list: - assert Property.NAME in suite_desc - assert Property.DESCRIPTION in suite_desc - assert Property.ABSPATH in suite_desc - assert Property.ABSPATH is not None - - def test_describe(self): - desc = Suite('suite-a').describe() - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc + with pytest.raises(TypeError) as excinfo: + Suite() + assert '__init__() takes exactly 2 arguments (1 given)' \ + in str(excinfo.value) diff --git a/tests/unit/runner/testplan_test.py b/tests/unit/runner/testplan_test.py deleted file mode 100644 index 7e42f557..00000000 --- a/tests/unit/runner/testplan_test.py +++ /dev/null @@ -1,48 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from os import path -import pytest - -from qtip.runner.testplan import TestPlan -from qtip.runner.benchmark import Property - - -class TestTestPlanClass: - def test_attr(self): - assert len(TestPlan._paths) is 1 - - -class TestTestPlan: - TestPlan._paths = [path.join(path.dirname(__file__), path.pardir, - path.pardir, 'data', 'testplan')] - - def test_init(self): - plan = TestPlan('plan-a') - assert plan.name == 'plan-a' - - with pytest.raises(TypeError) as excinfo: - TestPlan() - assert '__init__() takes exactly 2 arguments (1 given)' \ - in str(excinfo.value) - - def test_list(self): - plan_list = TestPlan.list_all() - assert len(list(plan_list)) is 5 - for desc in plan_list: - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc - assert Property.ABSPATH is not None - - def test_describe(self): - desc = TestPlan('plan-a').describe() - assert Property.NAME in desc - assert Property.DESCRIPTION in desc - assert Property.ABSPATH in desc diff --git a/tests/unit/utils/args_handler_test.py b/tests/unit/utils/args_handler_test.py index 5cbe766e..dceca1f5 100644 --- a/tests/unit/utils/args_handler_test.py +++ b/tests/unit/utils/args_handler_test.py @@ -11,8 +11,8 @@ import mock import qtip.utils.args_handler +@pytest.mark.xfail(reason="to be fixed") class TestClass: - @pytest.mark.skip("(yujunz) test fails") @pytest.mark.parametrize("test_input, expected", [ (['fuel', '/home', 'benchmarks/testplan/default/network/iperf_bm.yaml'], ['fuel', '/home', "iperf", diff --git a/tests/unit/utils/cli_test.py b/tests/unit/utils/cli_test.py index 13bb857b..0f3e4158 100644 --- a/tests/unit/utils/cli_test.py +++ b/tests/unit/utils/cli_test.py @@ -5,6 +5,7 @@ from qtip.utils.cli import Cli from os.path import expanduser +@pytest.mark.skip("TODO(yujunz) recover test after refactoring") class TestClass: @pytest.mark.parametrize("test_input, expected", [ (['-l', diff --git a/tests/unit/utils/create_zones_test.py b/tests/unit/utils/create_zones_test.py index 4a86b5cb..dcfff5ec 100644 --- a/tests/unit/utils/create_zones_test.py +++ b/tests/unit/utils/create_zones_test.py @@ -50,8 +50,8 @@ class NovaMock(MagicMock): aggregates = AggMock() +@pytest.mark.xfail(reason="unstable result") class TestClass: - @pytest.mark.skip("(yujunz) unstable result") @pytest.mark.parametrize("test_input, expected", [ (['compute1', 'compute2'], ['create:compute1:compute1', diff --git a/tests/unit/utils/env_setup_test.py b/tests/unit/utils/env_setup_test.py index c40801cc..dea48190 100644 --- a/tests/unit/utils/env_setup_test.py +++ b/tests/unit/utils/env_setup_test.py @@ -78,7 +78,7 @@ class TestClass: result = filecmp.cmp(get_output("hosts"), "config/hosts") assert result - @pytest.mark.skip("(yujunz) to be fixed") + @pytest.mark.skip("(yujunz) test hung") def test_ping(self, capfd): test_class = Env_setup() mock_ips = mock.Mock(return_value=["127.0.0.1", "10.20.0.29"]) |