diff options
author | Yujun Zhang <zhang.yujunz@zte.com.cn> | 2016-12-21 00:19:46 +0800 |
---|---|---|
committer | Yujun Zhang <zhang.yujunz@zte.com.cn> | 2016-12-27 09:45:34 +0800 |
commit | 929bcdf94d14062e042d9f9451c28315a18e808d (patch) | |
tree | d57ef7c074348fb654a9d4b7cea8b69241474040 /qtip | |
parent | 16cfb003cd0f1b0dbf903432ab794cda2c7a4123 (diff) |
Implment https://wiki.opnfv.org/display/qtip/Design
Note that some obsolete test cases are marked expected failure, will
be deprecated after architecture evolution.
JIRA: QTIP-148
Change-Id: I52bc9391569d516e298d9e659517161b4dce794a
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>
Diffstat (limited to 'qtip')
-rw-r--r-- | qtip/base/__init__.py | 0 | ||||
-rw-r--r-- | qtip/base/benchmark.py | 95 | ||||
-rw-r--r-- | qtip/cli/commands/cmd_plan.py (renamed from qtip/cli/commands/cmd_testplan.py) | 34 | ||||
-rw-r--r-- | qtip/runner/benchmark.py | 52 | ||||
-rw-r--r-- | qtip/runner/case.py (renamed from qtip/runner/testplan.py) | 16 | ||||
-rw-r--r-- | qtip/runner/plan.py | 26 | ||||
-rw-r--r-- | qtip/runner/suite.py | 22 | ||||
-rw-r--r-- | qtip/spec/__init__.py | 0 | ||||
-rw-r--r-- | qtip/spec/metric.py (renamed from qtip/runner/perftest.py) | 15 | ||||
-rw-r--r-- | qtip/spec/qpi.py | 18 |
10 files changed, 175 insertions, 103 deletions
diff --git a/qtip/base/__init__.py b/qtip/base/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/base/__init__.py diff --git a/qtip/base/benchmark.py b/qtip/base/benchmark.py new file mode 100644 index 00000000..b38e6016 --- /dev/null +++ b/qtip/base/benchmark.py @@ -0,0 +1,95 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from itertools import chain +from os import listdir +from os import path +import yaml + + +class Property(object): + # list + NAME = 'name' + CONTENT = 'content' + ABSPATH = 'abspath' + # content + TITLE = 'title' + DESCRIPTION = 'description' + # spec + ALGORITHM = 'algorithm' + SECTIONS = 'sections' + WEIGHT = 'weight' + METRICS = 'metrics' + SPEC = 'spec' + WORKLOADS = 'workloads' + # plan + INFO = 'info' + FACILITY = 'facility' + ENGINEER = 'engineer' + SUITES = 'suites' + # suite + QPI_SPEC = 'QPI_spec' + CONDITION = 'condition' + CASES = 'cases' + # case + METRIC_SPEC = 'metric_spec' + CONFIG = 'config' + + +class QtipError(Exception): + pass + + +class Algorithm(object): + ARITHMETIC_MEAN = 'arithmetic mean' + WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean' + GEOMETRIC_MEAN = 'geometric mean' + WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean' + + +ROOT_DIR = 'benchmarks' + + +class Benchmark(object): + """Abstract class of QTIP benchmarks""" + DEFAULT_DIR = '.' + _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, + ROOT_DIR)] + + def __init__(self, name, paths=None): + self._file = name + self._abspath = self._find(name, paths) + self.name = path.splitext(name)[0] + + def _find(self, name, paths): + """find a benchmark in searching paths""" + paths = self._paths if paths is None else paths + name = path.join(self.DEFAULT_DIR, name) + for p in paths: + abspath = path.join(p, name) + if path.exists(abspath): + return abspath + raise QtipError("'{}' not found in paths: {}".format(name, paths)) + + @classmethod + def list_all(cls, paths=None): + """list all available benchmarks""" + paths = cls._paths if paths is None else paths + names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR)) + for p in paths]) + for name in names: + item = cls(name, paths=paths) + yield { + Property.NAME: name, + Property.ABSPATH: item._abspath, + Property.CONTENT: item.content()} + + def content(self): + """description of benchmark""" + return yaml.safe_load(file(self._abspath)) diff --git a/qtip/cli/commands/cmd_testplan.py b/qtip/cli/commands/cmd_plan.py index 200e4665..01bf8251 100644 --- a/qtip/cli/commands/cmd_testplan.py +++ b/qtip/cli/commands/cmd_plan.py @@ -8,10 +8,8 @@ ############################################################################## import click -import sys from prettytable import PrettyTable - -from qtip.runner.testplan import TestPlan +from qtip.runner.plan import Plan @click.group() @@ -20,30 +18,26 @@ def cli(): @cli.group() -def testplan(): +def plan_cmd(): pass -@testplan.command('list', help='List the different TestPlans.') -def list(): - testplans = TestPlan.list_all() +@plan_cmd.command('list', help='List the different TestPlans.') +def list_all(): + plans = Plan.list_all() table = PrettyTable(["Testplans"]) table.align = 'l' - for testplan in testplans: - table.add_row([testplan['name']]) + for plan in plans: + table.add_row([plan['name']]) click.echo(table) -@testplan.command('show', help='Show details of specified TestPlan.') +@plan_cmd.command('show', help='Show details of specified TestPlan.') @click.argument('name') def show(name): - plan = TestPlan(name) - desc = plan.describe() - if desc['abspath'] is None: - click.echo("Wrong TestPlan specified") - sys.exit(1) - else: - table = PrettyTable(["Name", "Description"]) - table.align = 'l' - table.add_row([desc['name'], desc['description']]) - click.echo(table) + plan = Plan(name) + results = plan.content() + table = PrettyTable(["Name", "Description"]) + table.align = 'l' + table.add_row([results['name'], results['description']]) + click.echo(table) diff --git a/qtip/runner/benchmark.py b/qtip/runner/benchmark.py deleted file mode 100644 index 58e5a8d9..00000000 --- a/qtip/runner/benchmark.py +++ /dev/null @@ -1,52 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from itertools import chain -from os import listdir -from os import path - - -class Property: - NAME = 'name' - DESCRIPTION = 'description' - ABSPATH = 'abspath' - - -class Benchmark(object): - """Abstract class of QTIP benchmarks""" - - _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir, - 'benchmarks')] - - def __init__(self, name): - self.name = name - self._abspath = self._find(name) - - def _find(self, name): - """find a benchmark in searching paths""" - for p in self._paths: - abspath = path.join(p, name) - if path.exists(abspath): - return abspath - return None - - @classmethod - def list_all(cls): - """list all available benchmarks""" - names = chain.from_iterable([listdir(p) for p in cls._paths]) - return [cls(name).describe() for name in names] - - def describe(self): - """description of benchmark""" - # TODO(yujunz) read description from benchmark content - return { - Property.NAME: self.name, - Property.DESCRIPTION: 'QTIP benchmark', - Property.ABSPATH: self._abspath - } diff --git a/qtip/runner/testplan.py b/qtip/runner/case.py index f48a7147..eb3febc2 100644 --- a/qtip/runner/testplan.py +++ b/qtip/runner/case.py @@ -7,15 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from os import path +from qtip.base.benchmark import Property +from qtip.spec.metric import MetricSpec -from benchmark import Benchmark - -class TestPlan(Benchmark): - """WIP(yujunz): - a test plan is consist of test condition and several suites which can be - executed by user""" - - # paths to search for suites - _paths = [path.join(p, 'testplan') for p in Benchmark._paths] +class Case(object): + def __init__(self, spec, paths=None): + self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths) + self.config = spec[Property.CONFIG] diff --git a/qtip/runner/plan.py b/qtip/runner/plan.py new file mode 100644 index 00000000..265ad8d7 --- /dev/null +++ b/qtip/runner/plan.py @@ -0,0 +1,26 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.base.benchmark import Benchmark, Property +from qtip.runner.suite import Suite + + +class Plan(Benchmark): + """ + a benchmark plan is consist of basic information and several suites""" + + DEFAULT_DIR = 'plans' + + def __init__(self, name, paths=None): + super(Plan, self).__init__(name, paths=paths) + content = self.content() + + self.info = content[Property.INFO] + self.suites = [Suite(suite, paths=paths) + for suite in content[Property.SUITES]] diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py index 0086a20e..55033d2b 100644 --- a/qtip/runner/suite.py +++ b/qtip/runner/suite.py @@ -7,16 +7,16 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from os import path +from qtip.base.benchmark import Property +from qtip.spec.qpi import QPISpec +from qtip.runner.case import Case -from benchmark import Benchmark - -class Suite(Benchmark): - """WIP(yujunz): - a suite is consist of one or several perf tests and produces one QPI. - It must be executed as part of testplan - """ - - # paths to search for suites - _paths = [path.join(p, 'suite') for p in Benchmark._paths] +class Suite(object): + """a suite of benchmark cases under specified condition""" + def __init__(self, spec, paths=None): + self._paths = paths + self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths) + self.condition = spec.get(Property.CONDITION, {}) + self.cases = [Case(case_spec, paths) + for case_spec in spec.get(Property.CASES, [])] diff --git a/qtip/spec/__init__.py b/qtip/spec/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/spec/__init__.py diff --git a/qtip/runner/perftest.py b/qtip/spec/metric.py index a9b54716..e9c70547 100644 --- a/qtip/runner/perftest.py +++ b/qtip/spec/metric.py @@ -7,15 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from os import path +from qtip.base.benchmark import Benchmark -from benchmark import Benchmark - -class PerfTest(Benchmark): - """WIP(yujunz): - a perftest is the driver of external performance test tools - It is usually referred in a suite to collect performance metric""" - - # paths to search for perftest - _paths = [path.join(p, 'perftest') for p in Benchmark._paths] +class MetricSpec(Benchmark): + """metrics in QTIP are categorized by performance test tools, such as + dhrystone, whetstone and etc""" + DEFAULT_DIR = 'metrics' diff --git a/qtip/spec/qpi.py b/qtip/spec/qpi.py new file mode 100644 index 00000000..b7d7aa02 --- /dev/null +++ b/qtip/spec/qpi.py @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from qtip.base.benchmark import Benchmark + + +class QPISpec(Benchmark): + """ + a QPI specification defines how to calculate a performance index from + collected metrics. + """ + DEFAULT_DIR = 'QPI' |