summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.gitignore4
-rw-r--r--CONTRIBUTING.md1
-rw-r--r--DEVELOP.md101
-rw-r--r--benchmarks/suite/compute.yaml12
-rw-r--r--qtip/base/benchmark.py95
-rw-r--r--qtip/base/constant.py50
-rw-r--r--qtip/base/error.py36
-rw-r--r--qtip/cli/commands/cmd_plan.py3
-rw-r--r--qtip/collector/__init__.py (renamed from qtip/drivers/__init__.py)0
-rw-r--r--qtip/collector/base.py (renamed from qtip/drivers/base.py)4
-rw-r--r--qtip/collector/logfile.py (renamed from qtip/agent/reporter.py)7
-rw-r--r--qtip/driver/__init__.py (renamed from qtip/spec/__init__.py)0
-rw-r--r--qtip/driver/ansible.py (renamed from qtip/drivers/ansible.py)0
-rw-r--r--qtip/driver/base.py (renamed from qtip/spec/qpi.py)16
-rw-r--r--qtip/driver/sample.py15
-rw-r--r--qtip/driver/yardstick.py (renamed from qtip/drivers/yardstick.py)0
-rw-r--r--qtip/loader/__init__.py0
-rw-r--r--qtip/loader/base.py61
-rw-r--r--qtip/loader/metric.py (renamed from qtip/spec/metric.py)6
-rw-r--r--qtip/loader/plan.py (renamed from qtip/runner/plan.py)21
-rw-r--r--qtip/loader/qpi.py38
-rw-r--r--qtip/reporter/__init__.py0
-rw-r--r--qtip/reporter/base.py14
-rw-r--r--qtip/reporter/console.py (renamed from qtip/agent/collector.py)7
-rw-r--r--qtip/runner/__init__.py43
-rw-r--r--qtip/runner/case.py17
-rw-r--r--qtip/runner/suite.py22
-rw-r--r--qtip/utils/formula.py29
-rw-r--r--requirements.txt1
-rw-r--r--tests/conftest.py8
-rw-r--r--tests/data/benchmarks/QPI/compute.yaml37
-rw-r--r--tests/data/benchmarks/QPI/fake-qpi.yaml9
-rw-r--r--tests/data/benchmarks/metric/dhrystone.yaml9
-rw-r--r--tests/data/benchmarks/metric/dpi.yaml (renamed from tests/data/benchmarks/metrics/dpi.yaml)0
-rw-r--r--tests/data/benchmarks/metric/fake-metric.yaml7
-rw-r--r--tests/data/benchmarks/metric/ramspeed.yaml (renamed from tests/data/benchmarks/metrics/ramspeed.yaml)0
-rw-r--r--tests/data/benchmarks/metric/ssl.yaml (renamed from tests/data/benchmarks/metrics/ssl.yaml)0
-rw-r--r--tests/data/benchmarks/metric/whetstone.yaml10
-rw-r--r--tests/data/benchmarks/metrics/dhrystone.yaml7
-rw-r--r--tests/data/benchmarks/metrics/whetstone.yaml5
-rw-r--r--tests/data/benchmarks/plan/fake-plan.yaml10
-rw-r--r--tests/data/benchmarks/plans/verification.yaml21
-rw-r--r--tests/unit/loader/metric_test.py24
-rw-r--r--tests/unit/loader/plan_test.py (renamed from tests/unit/runner/plan_test.py)25
-rw-r--r--tests/unit/loader/qpi_test.py28
-rw-r--r--tests/unit/runner/case_test.py15
-rw-r--r--tests/unit/runner/conftest.py30
-rw-r--r--tests/unit/runner/suite_test.py27
48 files changed, 539 insertions, 336 deletions
diff --git a/.gitignore b/.gitignore
index aab79ed0..39d12173 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,3 +66,7 @@ target/
/docs_build/
/docs_output/
.idea
+
+# setup.py cache
+AUTHORS
+ChangeLog
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ad693d36..a4929172 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -37,6 +37,7 @@ Active Reviewers
Current list of active reviewers in gerrit group `qtip-reviewers`
+* Akhil Batra <akhil.batra@research.iiit.ac.in>
* Serena Feng <feng.xiaowei@zte.com.cn>
* Taseer Ahmed <taseer94@gmail.com>
* Yujun Zhang <zhang.yujunz@zte.com.cn>
diff --git a/DEVELOP.md b/DEVELOP.md
new file mode 100644
index 00000000..9ec3a93d
--- /dev/null
+++ b/DEVELOP.md
@@ -0,0 +1,101 @@
+# QTIP Developer Guide
+
+This guide is about how to **develop** QTIP.
+
+If you just want to use it for performance benchmark, check the user guide
+instead.
+
+## Getting Started
+
+### Source code
+
+```bash
+~$ git clone https://gerrit.opnfv.org/gerrit/qtip
+```
+
+### VirtualEnv
+
+It is recommended to use [virtualenv](https://virtualenv.pypa.io) to isolate
+your development environment from system, especially when you are working on
+several different projects.
+
+### Testing
+
+QTIP use [tox](https://tox.readthedocs.io) to automate the testing tasks
+
+```bash
+$ pip install tox
+$ tox
+```
+
+## Architecture
+
+**TODO**: move to design spec
+
+QTIP has a flexible architecture to allow different deployment mode
+
+- **Standalone**: full feature performance benchmark platform.
+- **Agent**: minimal agent driven by external test runners.
+
+### Standalone Mode (Solo)
+
+QTIP instance deployed in container, VM or host generate benchmark report and
+push data to Indices Hub for storage and visualization.
+
+![solo](https://wiki.opnfv.org/download/attachments/8687017/Standalone.png?api=v2)
+
+### Agent Mode (Melody)
+
+QTIP Collector and Reporter driven by external test framework or runner such as
+[yardstick](https://wiki.opnfv.org/display/yardstick),
+[pytest](http://doc.pytest.org/) and etc.
+
+![melody](https://wiki.opnfv.org/download/attachments/8687017/Agent.png?api=v2)
+
+## Core Modules
+
+TBD
+
+- loader
+- runner
+- collector
+- reporter
+
+## Drivers
+
+TBD
+
+- ansible
+- yardstick
+
+## Interfaces
+
+### Agent
+
+TBD
+
+### CLI
+
+TBD
+
+### API
+
+TBD
+
+## Assets
+
+**TODO**: move to user guide
+
+- benchmark plan
+- QPI spec
+- metric spec
+
+## Docker Image
+
+TBD
+
+## Annex
+
+### Directories
+
+TBD
diff --git a/benchmarks/suite/compute.yaml b/benchmarks/suite/compute.yaml
index 0761a87b..197d5720 100644
--- a/benchmarks/suite/compute.yaml
+++ b/benchmarks/suite/compute.yaml
@@ -1,12 +1,12 @@
QPI: compute
description: sample performance index of computing
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
section:
- name: Integer
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: dhrystone
workloads:
@@ -14,7 +14,7 @@ section:
- multi_cpu
- name: Floating
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: whetstone
workloads:
@@ -22,7 +22,7 @@ section:
- multi_cpu
- name: Memory
weight: 0.2
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: ramspeed
workloads:
@@ -30,7 +30,7 @@ section:
- float: [add, average, copy, scale, triad]
- name: DPI
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: dpi
workloads:
@@ -38,7 +38,7 @@ section:
- pps
- name: SSL
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: ssl
workloads:
diff --git a/qtip/base/benchmark.py b/qtip/base/benchmark.py
deleted file mode 100644
index b38e6016..00000000
--- a/qtip/base/benchmark.py
+++ /dev/null
@@ -1,95 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from itertools import chain
-from os import listdir
-from os import path
-import yaml
-
-
-class Property(object):
- # list
- NAME = 'name'
- CONTENT = 'content'
- ABSPATH = 'abspath'
- # content
- TITLE = 'title'
- DESCRIPTION = 'description'
- # spec
- ALGORITHM = 'algorithm'
- SECTIONS = 'sections'
- WEIGHT = 'weight'
- METRICS = 'metrics'
- SPEC = 'spec'
- WORKLOADS = 'workloads'
- # plan
- INFO = 'info'
- FACILITY = 'facility'
- ENGINEER = 'engineer'
- SUITES = 'suites'
- # suite
- QPI_SPEC = 'QPI_spec'
- CONDITION = 'condition'
- CASES = 'cases'
- # case
- METRIC_SPEC = 'metric_spec'
- CONFIG = 'config'
-
-
-class QtipError(Exception):
- pass
-
-
-class Algorithm(object):
- ARITHMETIC_MEAN = 'arithmetic mean'
- WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
- GEOMETRIC_MEAN = 'geometric mean'
- WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
-
-
-ROOT_DIR = 'benchmarks'
-
-
-class Benchmark(object):
- """Abstract class of QTIP benchmarks"""
- DEFAULT_DIR = '.'
- _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
- ROOT_DIR)]
-
- def __init__(self, name, paths=None):
- self._file = name
- self._abspath = self._find(name, paths)
- self.name = path.splitext(name)[0]
-
- def _find(self, name, paths):
- """find a benchmark in searching paths"""
- paths = self._paths if paths is None else paths
- name = path.join(self.DEFAULT_DIR, name)
- for p in paths:
- abspath = path.join(p, name)
- if path.exists(abspath):
- return abspath
- raise QtipError("'{}' not found in paths: {}".format(name, paths))
-
- @classmethod
- def list_all(cls, paths=None):
- """list all available benchmarks"""
- paths = cls._paths if paths is None else paths
- names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR))
- for p in paths])
- for name in names:
- item = cls(name, paths=paths)
- yield {
- Property.NAME: name,
- Property.ABSPATH: item._abspath,
- Property.CONTENT: item.content()}
-
- def content(self):
- """description of benchmark"""
- return yaml.safe_load(file(self._abspath))
diff --git a/qtip/base/constant.py b/qtip/base/constant.py
new file mode 100644
index 00000000..76481b47
--- /dev/null
+++ b/qtip/base/constant.py
@@ -0,0 +1,50 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class FormulaName(object):
+ """formula names"""
+ ARITHMETIC_MEAN = 'arithmetic mean'
+ WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
+ GEOMETRIC_MEAN = 'geometric mean'
+ WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
+
+
+class PkgName(object):
+ """QTIP package names"""
+ COLLECTOR = 'collector'
+ DRIVER = 'driver'
+ REPORTER = 'reporter'
+ RUNNER = 'runner'
+ SPEC = 'spec'
+
+
+class PropName(object):
+ """property names"""
+ # list
+ NAME = 'name'
+ CONTENT = 'content'
+ ABSPATH = 'abspath'
+ # content
+ name = 'name'
+ DESCRIPTION = 'description'
+ # spec
+ SECTIONS = 'sections'
+ WEIGHT = 'weight'
+ FORMULA = 'formula'
+ METRICS = 'metrics'
+ WORKLOADS = 'workloads'
+ # plan
+ CONFIG = 'config'
+ FACILITY = 'facility'
+ ENGINEER = 'engineer'
+ DRIVER = 'driver'
+ COLLECTOR = 'collector'
+ REPORTER = 'reporter'
+ QPIS = 'QPIs'
diff --git a/qtip/base/error.py b/qtip/base/error.py
new file mode 100644
index 00000000..01a7f7a6
--- /dev/null
+++ b/qtip/base/error.py
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class QtipError(Exception):
+ pass
+
+
+class InvalidFormat(QtipError):
+ def __init__(self, filename):
+ self.filename = filename
+
+
+class NotFound(QtipError):
+ def __init__(self, module, package='qtip'):
+ self.package = package
+ self.module = module
+
+
+class ToBeDoneError(QtipError):
+ """something still to be done"""
+ def __init__(self, method, module):
+ self.method = method
+ self.module = module
+
+
+def make_tbd(method, module='qtip'):
+ def tbd():
+ raise ToBeDoneError(method, module)
+ return tbd
diff --git a/qtip/cli/commands/cmd_plan.py b/qtip/cli/commands/cmd_plan.py
index 01bf8251..6f622e5a 100644
--- a/qtip/cli/commands/cmd_plan.py
+++ b/qtip/cli/commands/cmd_plan.py
@@ -9,7 +9,8 @@
import click
from prettytable import PrettyTable
-from qtip.runner.plan import Plan
+
+from qtip.loader.plan import Plan
@click.group()
diff --git a/qtip/drivers/__init__.py b/qtip/collector/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/drivers/__init__.py
+++ b/qtip/collector/__init__.py
diff --git a/qtip/drivers/base.py b/qtip/collector/base.py
index 1aa8d8ad..cd8fc797 100644
--- a/qtip/drivers/base.py
+++ b/qtip/collector/base.py
@@ -8,5 +8,5 @@
##############################################################################
-class BaseDriver(object):
- """performance testing tool driver"""
+class BaseCollector(object):
+ """performance metrics collector"""
diff --git a/qtip/agent/reporter.py b/qtip/collector/logfile.py
index b5c4acfa..6528ea9f 100644
--- a/qtip/agent/reporter.py
+++ b/qtip/collector/logfile.py
@@ -7,7 +7,8 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+from base import BaseCollector
-class Reporter(object):
- """generate test report and push test data to database"""
- pass
+
+class LogfileCollector(BaseCollector):
+ """collect performance metrics from log files"""
diff --git a/qtip/spec/__init__.py b/qtip/driver/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/spec/__init__.py
+++ b/qtip/driver/__init__.py
diff --git a/qtip/drivers/ansible.py b/qtip/driver/ansible.py
index 04e9f9bd..04e9f9bd 100644
--- a/qtip/drivers/ansible.py
+++ b/qtip/driver/ansible.py
diff --git a/qtip/spec/qpi.py b/qtip/driver/base.py
index b7d7aa02..6f5cab3c 100644
--- a/qtip/spec/qpi.py
+++ b/qtip/driver/base.py
@@ -7,12 +7,14 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark
+class BaseDriver(object):
+ """performance testing tool driver"""
+ def pre_run(self):
+ pass
-class QPISpec(Benchmark):
- """
- a QPI specification defines how to calculate a performance index from
- collected metrics.
- """
- DEFAULT_DIR = 'QPI'
+ def run(self):
+ pass
+
+ def post_run(self):
+ pass
diff --git a/qtip/driver/sample.py b/qtip/driver/sample.py
new file mode 100644
index 00000000..9b347949
--- /dev/null
+++ b/qtip/driver/sample.py
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from base import BaseDriver
+
+
+class SampleDriver(BaseDriver):
+ """sample driver that generates random data for testing"""
diff --git a/qtip/drivers/yardstick.py b/qtip/driver/yardstick.py
index 83f1b3d8..83f1b3d8 100644
--- a/qtip/drivers/yardstick.py
+++ b/qtip/driver/yardstick.py
diff --git a/qtip/loader/__init__.py b/qtip/loader/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/loader/__init__.py
diff --git a/qtip/loader/base.py b/qtip/loader/base.py
new file mode 100644
index 00000000..2f5ab67a
--- /dev/null
+++ b/qtip/loader/base.py
@@ -0,0 +1,61 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from itertools import chain
+from os import listdir
+from os import path
+import yaml
+
+from qtip.base.error import InvalidFormat, NotFound
+from qtip.base.constant import PropName
+
+ROOT_DIR = 'benchmarks'
+
+
+class BaseLoader(object):
+ """Abstract class of QTIP benchmark loader"""
+ RELATIVE_PATH = '.'
+ _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
+ ROOT_DIR)]
+
+ def __init__(self, name, paths=None):
+ self._file = name
+ self._abspath = self._find(name, paths=paths)
+
+ try:
+ content = yaml.safe_load(file(self._abspath))
+ except yaml.YAMLError:
+ # TODO(yujunz) log yaml error
+ raise InvalidFormat(self._abspath)
+
+ self.name = content[PropName.NAME] if PropName.NAME in content \
+ else path.splitext(name)[0]
+ self.content = content
+
+ def _find(self, name, paths=None):
+ """find a benchmark in searching paths"""
+ paths = self._paths if paths is None else paths
+ for p in paths:
+ abspath = path.join(p, self.RELATIVE_PATH, name)
+ if path.exists(abspath):
+ return abspath
+ raise NotFound(name, paths)
+
+ @classmethod
+ def list_all(cls, paths=None):
+ """list all available benchmarks"""
+ paths = cls._paths if paths is None else paths
+ names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH))
+ for p in paths])
+ for name in names:
+ item = cls(name, paths=paths)
+ yield {
+ PropName.NAME: name,
+ PropName.ABSPATH: item._abspath,
+ PropName.CONTENT: item.content}
diff --git a/qtip/spec/metric.py b/qtip/loader/metric.py
index e9c70547..8b6fa5d3 100644
--- a/qtip/spec/metric.py
+++ b/qtip/loader/metric.py
@@ -7,10 +7,10 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark
+from base import BaseLoader
-class MetricSpec(Benchmark):
+class MetricSpec(BaseLoader):
"""metrics in QTIP are categorized by performance test tools, such as
dhrystone, whetstone and etc"""
- DEFAULT_DIR = 'metrics'
+ RELATIVE_PATH = 'metric'
diff --git a/qtip/runner/plan.py b/qtip/loader/plan.py
index 265ad8d7..cf517ea2 100644
--- a/qtip/runner/plan.py
+++ b/qtip/loader/plan.py
@@ -7,20 +7,21 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark, Property
-from qtip.runner.suite import Suite
+from qtip.base.constant import PropName
+from qtip.loader.base import BaseLoader
+from qtip.loader.qpi import QPISpec
-class Plan(Benchmark):
+
+class Plan(BaseLoader):
+ """
+ a benchmark plan is consist of configuration and a QPI list
"""
- a benchmark plan is consist of basic information and several suites"""
- DEFAULT_DIR = 'plans'
+ RELATIVE_PATH = 'plan'
def __init__(self, name, paths=None):
- super(Plan, self).__init__(name, paths=paths)
- content = self.content()
+ super(Plan, self).__init__(name, paths)
- self.info = content[Property.INFO]
- self.suites = [Suite(suite, paths=paths)
- for suite in content[Property.SUITES]]
+ self.qpis = [QPISpec(qpi, paths=paths)
+ for qpi in self.content[PropName.QPIS]]
diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py
new file mode 100644
index 00000000..cfa918c5
--- /dev/null
+++ b/qtip/loader/qpi.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from base import BaseLoader
+from metric import MetricSpec
+
+from qtip.base.constant import PropName
+from qtip.utils.formula import Formula
+
+
+class QPISpec(BaseLoader):
+ """
+ a QPI specification defines how to calculate a performance index from
+ collected metrics.
+ """
+ RELATIVE_PATH = 'QPI'
+
+ def __init__(self, name, paths=None):
+ super(QPISpec, self).__init__(name, paths=paths)
+ content = self.content
+ self.formula = Formula(content[PropName.FORMULA])
+ self.sections = [Section(record, paths=paths)
+ for record in content[PropName.SECTIONS]]
+
+
+class Section(object):
+ def __init__(self, content, paths=None):
+ self.name = content[PropName.NAME]
+ self.weight = content[PropName.WEIGHT]
+ self.formula = Formula(content[PropName.FORMULA])
+ self.metrics = [MetricSpec(record, paths=paths)
+ for record in content[PropName.METRICS]]
diff --git a/qtip/reporter/__init__.py b/qtip/reporter/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/reporter/__init__.py
diff --git a/qtip/reporter/base.py b/qtip/reporter/base.py
new file mode 100644
index 00000000..b931d14d
--- /dev/null
+++ b/qtip/reporter/base.py
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseReporter(object):
+ """benchmark result reporter"""
+ def __init__(self, collector=None):
+ self.collector = collector
diff --git a/qtip/agent/collector.py b/qtip/reporter/console.py
index 3df21379..136ce358 100644
--- a/qtip/agent/collector.py
+++ b/qtip/reporter/console.py
@@ -7,7 +7,8 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+from base import BaseReporter
-class Collector(object):
- """collect test result and test condition"""
- pass
+
+class ConsoleReporter(BaseReporter):
+ """report result to console"""
diff --git a/qtip/runner/__init__.py b/qtip/runner/__init__.py
index e69de29b..eab81156 100644
--- a/qtip/runner/__init__.py
+++ b/qtip/runner/__init__.py
@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.constant import PkgName, PropName
+from qtip.base.error import NotFound
+from qtip.collector.stdout import StdoutCollector
+from qtip.driver.random import RandomDriver
+from qtip.reporter.console import ConsoleReporter
+
+
+class Runner(object):
+ def __init__(self, spec, config=None):
+ if config is None:
+ config = spec[PropName.CONFIG]
+
+ driver_name = config[PropName.DRIVER]
+ collector_name = config[PropName.COLLECTOR]
+ reporter_name = config[PropName.REPORTER]
+
+ # TODO(yujunz) dynamically load modules by name
+
+ if driver_name == 'random':
+ self.driver = RandomDriver()
+ else:
+ raise NotFound(driver_name, package=PkgName.DRIVER)
+
+ if collector_name == 'stdout':
+ self.collector = StdoutCollector()
+ else:
+ raise NotFound(collector_name,
+ package=PkgName.COLLECTOR)
+
+ if reporter_name == 'console':
+ self.reporter = ConsoleReporter()
+ else:
+ raise NotFound(reporter_name,
+ package=PkgName.REPORTER)
diff --git a/qtip/runner/case.py b/qtip/runner/case.py
deleted file mode 100644
index eb3febc2..00000000
--- a/qtip/runner/case.py
+++ /dev/null
@@ -1,17 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
-
-
-class Case(object):
- def __init__(self, spec, paths=None):
- self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths)
- self.config = spec[Property.CONFIG]
diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py
deleted file mode 100644
index 55033d2b..00000000
--- a/qtip/runner/suite.py
+++ /dev/null
@@ -1,22 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.benchmark import Property
-from qtip.spec.qpi import QPISpec
-from qtip.runner.case import Case
-
-
-class Suite(object):
- """a suite of benchmark cases under specified condition"""
- def __init__(self, spec, paths=None):
- self._paths = paths
- self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths)
- self.condition = spec.get(Property.CONDITION, {})
- self.cases = [Case(case_spec, paths)
- for case_spec in spec.get(Property.CASES, [])]
diff --git a/qtip/utils/formula.py b/qtip/utils/formula.py
new file mode 100644
index 00000000..cdfbae86
--- /dev/null
+++ b/qtip/utils/formula.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import numpy
+
+from qtip.base.error import make_tbd
+from qtip.base.constant import FormulaName
+
+
+MAPPING = {
+ FormulaName.ARITHMETIC_MEAN: numpy.mean,
+ FormulaName.WEIGHTED_ARITHMETIC_MEAN: numpy.average,
+ # TODO(yujunz) find or implement the method
+ FormulaName.GEOMETRIC_MEAN: make_tbd(FormulaName.GEOMETRIC_MEAN, __name__),
+ # TODO(yujunz) find or implement the method
+ FormulaName.WEIGHTED_GEOMETRIC_MEAN:
+ make_tbd(FormulaName.GEOMETRIC_MEAN, __name__)}
+
+
+class Formula:
+ """calculate a score from give data"""
+ def __init__(self, name):
+ self.calculate = MAPPING[name]
diff --git a/requirements.txt b/requirements.txt
index cdaeef70..ec566b83 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,4 @@ Flask==0.11.1
Flask-RESTful==0.3.5
flask-restful-swagger==0.19
ansible==2.1.1.0
+numpy==1.11.3
diff --git a/tests/conftest.py b/tests/conftest.py
index 8e2ecf7c..7acb75e6 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,8 +8,11 @@
##############################################################################
from os import path
+
import pytest
+from qtip.loader.plan import Plan
+
@pytest.fixture(scope='session')
def data_root():
@@ -19,3 +22,8 @@ def data_root():
@pytest.fixture(scope='session')
def benchmarks_root(data_root):
return path.join(data_root, 'benchmarks')
+
+
+@pytest.fixture(scope='session')
+def plan(benchmarks_root):
+ return Plan('fake-plan.yaml', [benchmarks_root])
diff --git a/tests/data/benchmarks/QPI/compute.yaml b/tests/data/benchmarks/QPI/compute.yaml
index 5ad8a4b6..e28efaf9 100644
--- a/tests/data/benchmarks/QPI/compute.yaml
+++ b/tests/data/benchmarks/QPI/compute.yaml
@@ -1,44 +1,29 @@
title: compute
description: sample performance index of computing
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
sections:
- name: Integer
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dhrystone.yaml
- workloads:
- - single_cpu
- - multi_cpu
+ - dhrystone.yaml
- name: Float
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dhrystone.yaml
- workloads:
- - single_cpu
- - multi_cpu
+ - whetstone.yaml
- name: Memory
weight: 0.2
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: ramspeed.yaml
- workloads:
- - int: [add, average, copy, scale, triad]
- - float: [add, average, copy, scale, triad]
+ - ramspeed.yaml
- name: DPI
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dpi.yaml
- workloads:
- - bps
- - pps
+ - dpi.yaml
- name: SSL
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: ssl.yaml
- workloads:
- - aes_128_cbc: [512, 1024, 2048, 4096]
- - rsa_sig: [16, 64, 256, 1024, 8192]
+ - ssl.yaml
diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml
new file mode 100644
index 00000000..aa1097f4
--- /dev/null
+++ b/tests/data/benchmarks/QPI/fake-qpi.yaml
@@ -0,0 +1,9 @@
+name: Fake QPI
+description: a fake QPI producing random result
+formula: weighted arithmetic mean
+sections:
+- name: Fake Section
+ weight: 0.5
+ formula: geometric mean
+ metrics:
+ - fake-metric.yaml
diff --git a/tests/data/benchmarks/metric/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml
new file mode 100644
index 00000000..220b7841
--- /dev/null
+++ b/tests/data/benchmarks/metric/dhrystone.yaml
@@ -0,0 +1,9 @@
+name: dhrystone
+description: >
+ A synthetic computing benchmark program intended to be representative of
+ system (integer) programming.
+links:
+ - https://en.wikipedia.org/wiki/Dhrystone
+workloads:
+ - single_cpu
+ - multi_cpu
diff --git a/tests/data/benchmarks/metrics/dpi.yaml b/tests/data/benchmarks/metric/dpi.yaml
index fc24c8d1..fc24c8d1 100644
--- a/tests/data/benchmarks/metrics/dpi.yaml
+++ b/tests/data/benchmarks/metric/dpi.yaml
diff --git a/tests/data/benchmarks/metric/fake-metric.yaml b/tests/data/benchmarks/metric/fake-metric.yaml
new file mode 100644
index 00000000..c5fcc5c2
--- /dev/null
+++ b/tests/data/benchmarks/metric/fake-metric.yaml
@@ -0,0 +1,7 @@
+name: fake compute
+description: >
+ a fake compute performance index which generate random result
+workloads: # all supported workloads
+ - fake_workload_1
+ - fake_workload_2
+ - fake_workload_3
diff --git a/tests/data/benchmarks/metrics/ramspeed.yaml b/tests/data/benchmarks/metric/ramspeed.yaml
index bb7618b7..bb7618b7 100644
--- a/tests/data/benchmarks/metrics/ramspeed.yaml
+++ b/tests/data/benchmarks/metric/ramspeed.yaml
diff --git a/tests/data/benchmarks/metrics/ssl.yaml b/tests/data/benchmarks/metric/ssl.yaml
index 21e8add7..21e8add7 100644
--- a/tests/data/benchmarks/metrics/ssl.yaml
+++ b/tests/data/benchmarks/metric/ssl.yaml
diff --git a/tests/data/benchmarks/metric/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml
new file mode 100644
index 00000000..448c9645
--- /dev/null
+++ b/tests/data/benchmarks/metric/whetstone.yaml
@@ -0,0 +1,10 @@
+name: whetstone
+description: >
+ A synthetic benchmark for evaluating the performance of computers.
+ The Whetstone benchmark primarily measures the floating-point arithmetic
+ performance.
+links:
+ - https://en.wikipedia.org/wiki/Whetstone_(benchmark)
+workloads:
+ - single_cpu
+ - multi_cpu
diff --git a/tests/data/benchmarks/metrics/dhrystone.yaml b/tests/data/benchmarks/metrics/dhrystone.yaml
deleted file mode 100644
index b0d55ed2..00000000
--- a/tests/data/benchmarks/metrics/dhrystone.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-name: dhrystone
-description: >
- a synthetic computing benchmark program intended to be representative of
- system (integer) programming
-workloads:
- - single_cpu
- - multi_cpu
diff --git a/tests/data/benchmarks/metrics/whetstone.yaml b/tests/data/benchmarks/metrics/whetstone.yaml
deleted file mode 100644
index d83680c4..00000000
--- a/tests/data/benchmarks/metrics/whetstone.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-name: dhrystone
-description: a synthetic benchmark for evaluating the performance of computers
-workloads:
- - single_cpu
- - multi_cpu
diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml
new file mode 100644
index 00000000..8887f66d
--- /dev/null
+++ b/tests/data/benchmarks/plan/fake-plan.yaml
@@ -0,0 +1,10 @@
+name: fake plan
+description: fake benchmark plan for demonstration and testing
+config:
+ facility: local
+ engineer: local
+ driver: sample
+ collector: logfile
+ reporter: console
+QPIs:
+ - fake-qpi.yaml
diff --git a/tests/data/benchmarks/plans/verification.yaml b/tests/data/benchmarks/plans/verification.yaml
deleted file mode 100644
index b146ee37..00000000
--- a/tests/data/benchmarks/plans/verification.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-title: verification
-description: benchmark plan for QTIP verification
-info:
- facility: opnfv-ci
- engineer: opnfv-bot
-suites:
- - QPI_spec: compute.yaml
-# Uncomment next line to setup suite running condition
-# condition: {}
-# Uncomment the following lines setup case running configuration
-# cases:
-# - metric_spec: dhrystone.yaml
-# config: {}
-# - metric_spec: whetstone.yaml
-# config: {}
-# - metric_spec: ramspeed.yaml
-# config: {}
-# - metric_spec: dpi.yaml
-# config: {}
-# - metric_spec: ssl.yaml
-# config: {}
diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py
index 5eced700..d2be0388 100644
--- a/tests/unit/loader/metric_test.py
+++ b/tests/unit/loader/metric_test.py
@@ -9,8 +9,8 @@
import pytest
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
+from qtip.base.constant import PropName
+from qtip.loader.metric import MetricSpec
@pytest.fixture(scope='module')
@@ -29,17 +29,17 @@ def init_test(metric_spec):
def list_all_test():
metric_list = MetricSpec.list_all()
- assert len(list(metric_list)) is 1
+ assert len(list(metric_list)) is 6
for desc in metric_list:
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
+ assert PropName.NAME in desc
+ assert PropName.DESCRIPTION in desc
+ assert PropName.ABSPATH in desc
+ assert PropName.ABSPATH is not None
def content_test(metric):
- content = metric.content()
- assert Property.NAME in content
- assert Property.DESCRIPTION in content
- assert Property.WORKLOADS in content
- assert isinstance(content[Property.WORKLOADS], list)
+ content = metric.content
+ assert PropName.NAME in content
+ assert PropName.DESCRIPTION in content
+ assert PropName.WORKLOADS in content
+ assert isinstance(content[PropName.WORKLOADS], list)
diff --git a/tests/unit/runner/plan_test.py b/tests/unit/loader/plan_test.py
index d783e5e9..6aab5e8a 100644
--- a/tests/unit/runner/plan_test.py
+++ b/tests/unit/loader/plan_test.py
@@ -9,12 +9,15 @@
import pytest
-from qtip.base.benchmark import Property
-from qtip.runner.plan import Plan
+from qtip.base.constant import PropName
+from qtip.loader.plan import Plan, QPISpec
def test_init(plan):
- assert plan.name == 'verification'
+ assert plan.name == 'fake plan'
+ assert isinstance(plan.content, dict)
+ for qpi in plan.qpis:
+ assert isinstance(qpi, QPISpec)
with pytest.raises(TypeError) as excinfo:
Plan()
@@ -26,13 +29,15 @@ def test_list_all(benchmarks_root):
plan_list = Plan.list_all(paths=[benchmarks_root])
assert len(list(plan_list)) is 1
for desc in plan_list:
- assert Property.NAME in desc
- assert Property.CONTENT in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
+ assert PropName.NAME in desc
+ assert PropName.CONTENT in desc
+ assert PropName.ABSPATH in desc
+ assert PropName.ABSPATH is not None
def test_content(plan):
- content = plan.content()
- assert Property.TITLE in content
- assert Property.DESCRIPTION in content
+ content = plan.content
+ assert PropName.NAME in content
+ assert PropName.DESCRIPTION in content
+ assert PropName.CONFIG in content
+ assert PropName.QPIS in content
diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py
index bfa1f580..4b3fd4d0 100644
--- a/tests/unit/loader/qpi_test.py
+++ b/tests/unit/loader/qpi_test.py
@@ -9,8 +9,8 @@
import pytest
-from qtip.base.benchmark import Algorithm, Property
-from qtip.spec.qpi import QPISpec
+from qtip.base.constant import FormulaName, PropName
+from qtip.loader.qpi import QPISpec
QPI_SPEC = 'compute.yaml'
@@ -31,22 +31,22 @@ def test_init(qpi_spec):
def test_list_all(benchmarks_root):
qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root])
- assert len(list(qpi_spec_list)) is 1
+ assert len(list(qpi_spec_list)) is 2
for item in qpi_spec_list:
- assert Property.NAME in item
- assert Property.CONTENT in item
- assert Property.ABSPATH in item
- assert Property.ABSPATH is not None
+ assert PropName.NAME in item
+ assert PropName.CONTENT in item
+ assert PropName.ABSPATH in item
+ assert PropName.ABSPATH is not None
def test_content(qpi_spec):
- content = qpi_spec.content()
- assert Property.DESCRIPTION in content
- assert Property.ALGORITHM in content
- assert Property.SECTIONS in content
+ content = qpi_spec.content
+ assert PropName.DESCRIPTION in content
+ assert PropName.FORMULA in content
+ assert PropName.SECTIONS in content
- assert content[Property.ALGORITHM] in Algorithm.__dict__.values()
- sections = content[Property.SECTIONS]
+ assert content[PropName.FORMULA] in FormulaName.__dict__.values()
+ sections = content[PropName.SECTIONS]
assert isinstance(sections, list)
for section in sections:
- assert Property.NAME in section
+ assert PropName.NAME in section
diff --git a/tests/unit/runner/case_test.py b/tests/unit/runner/case_test.py
deleted file mode 100644
index 59a54a84..00000000
--- a/tests/unit/runner/case_test.py
+++ /dev/null
@@ -1,15 +0,0 @@
-###############################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.spec.metric import MetricSpec
-
-
-def init_test(case):
- assert isinstance(case.metric_spec, MetricSpec)
- assert isinstance(case.config, dict)
diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py
deleted file mode 100644
index 6d14f7ae..00000000
--- a/tests/unit/runner/conftest.py
+++ /dev/null
@@ -1,30 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.base.benchmark import Property
-from qtip.runner.case import Case
-from qtip.runner.plan import Plan
-from qtip.runner.suite import Suite
-
-
-@pytest.fixture(scope='module')
-def plan(benchmarks_root):
- return Plan('verification.yaml', paths=[benchmarks_root])
-
-
-@pytest.fixture(scope='module')
-def suite(plan):
- return Suite(plan[Property.SUITES][0])
-
-
-@pytest.fixture(scope='module')
-def case(suite):
- return Case(suite[Property.CASES][0])
diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py
deleted file mode 100644
index 7dad8f62..00000000
--- a/tests/unit/runner/suite_test.py
+++ /dev/null
@@ -1,27 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.runner.suite import Suite
-from qtip.runner.case import Case
-from qtip.spec.qpi import QPISpec
-
-
-def init_test(suite):
- assert isinstance(suite.qpi, QPISpec)
- assert isinstance(suite.condition, dict)
- assert isinstance(suite.cases, list)
- for case in suite.cases:
- assert isinstance(case, Case)
-
- with pytest.raises(TypeError) as excinfo:
- Suite()
- assert '__init__() takes exactly 2 arguments (1 given)' \
- in str(excinfo.value)