summaryrefslogtreecommitdiffstats
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rw-r--r--tests/conftest.py7
-rw-r--r--tests/data/benchmarks/QPI/fake-qpi.yaml9
-rw-r--r--tests/data/benchmarks/metric/dhrystone.yaml (renamed from tests/data/benchmarks/metrics/dhrystone.yaml)0
-rw-r--r--tests/data/benchmarks/metric/dpi.yaml (renamed from tests/data/benchmarks/metrics/dpi.yaml)0
-rw-r--r--tests/data/benchmarks/metric/fake-metric.yaml7
-rw-r--r--tests/data/benchmarks/metric/ramspeed.yaml (renamed from tests/data/benchmarks/metrics/ramspeed.yaml)0
-rw-r--r--tests/data/benchmarks/metric/ssl.yaml (renamed from tests/data/benchmarks/metrics/ssl.yaml)0
-rw-r--r--tests/data/benchmarks/metric/whetstone.yaml (renamed from tests/data/benchmarks/metrics/whetstone.yaml)0
-rw-r--r--tests/data/benchmarks/plan/fake-plan.yaml10
-rw-r--r--tests/data/benchmarks/plans/verification.yaml21
-rw-r--r--tests/unit/cli/test_module.py31
-rw-r--r--tests/unit/cli/test_options.py32
-rw-r--r--tests/unit/loader/metric_test.py24
-rw-r--r--tests/unit/loader/qpi_test.py28
-rw-r--r--tests/unit/runner/case_test.py15
-rw-r--r--tests/unit/runner/conftest.py30
-rw-r--r--tests/unit/runner/plan_test.py18
-rw-r--r--tests/unit/runner/suite_test.py27
18 files changed, 100 insertions, 159 deletions
diff --git a/tests/conftest.py b/tests/conftest.py
index 8e2ecf7c..f1ec91fb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -10,6 +10,8 @@
from os import path
import pytest
+from qtip.runner.plan import Plan
+
@pytest.fixture(scope='session')
def data_root():
@@ -19,3 +21,8 @@ def data_root():
@pytest.fixture(scope='session')
def benchmarks_root(data_root):
return path.join(data_root, 'benchmarks')
+
+
+@pytest.fixture(scope='session')
+def plan(benchmarks_root):
+ return Plan('fake-plan.yaml', [benchmarks_root])
diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml
new file mode 100644
index 00000000..d75c6568
--- /dev/null
+++ b/tests/data/benchmarks/QPI/fake-qpi.yaml
@@ -0,0 +1,9 @@
+name: Fake QPI
+description: a fake QPI producing random result
+algorithm: weighted arithmetic mean
+sections:
+- name: Fake Section
+ weight: 0.5
+ algorithm: geometric mean
+ metrics:
+ - fake_metric.yaml
diff --git a/tests/data/benchmarks/metrics/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml
index b0d55ed2..b0d55ed2 100644
--- a/tests/data/benchmarks/metrics/dhrystone.yaml
+++ b/tests/data/benchmarks/metric/dhrystone.yaml
diff --git a/tests/data/benchmarks/metrics/dpi.yaml b/tests/data/benchmarks/metric/dpi.yaml
index fc24c8d1..fc24c8d1 100644
--- a/tests/data/benchmarks/metrics/dpi.yaml
+++ b/tests/data/benchmarks/metric/dpi.yaml
diff --git a/tests/data/benchmarks/metric/fake-metric.yaml b/tests/data/benchmarks/metric/fake-metric.yaml
new file mode 100644
index 00000000..c5fcc5c2
--- /dev/null
+++ b/tests/data/benchmarks/metric/fake-metric.yaml
@@ -0,0 +1,7 @@
+name: fake compute
+description: >
+ a fake compute performance index which generate random result
+workloads: # all supported workloads
+ - fake_workload_1
+ - fake_workload_2
+ - fake_workload_3
diff --git a/tests/data/benchmarks/metrics/ramspeed.yaml b/tests/data/benchmarks/metric/ramspeed.yaml
index bb7618b7..bb7618b7 100644
--- a/tests/data/benchmarks/metrics/ramspeed.yaml
+++ b/tests/data/benchmarks/metric/ramspeed.yaml
diff --git a/tests/data/benchmarks/metrics/ssl.yaml b/tests/data/benchmarks/metric/ssl.yaml
index 21e8add7..21e8add7 100644
--- a/tests/data/benchmarks/metrics/ssl.yaml
+++ b/tests/data/benchmarks/metric/ssl.yaml
diff --git a/tests/data/benchmarks/metrics/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml
index d83680c4..d83680c4 100644
--- a/tests/data/benchmarks/metrics/whetstone.yaml
+++ b/tests/data/benchmarks/metric/whetstone.yaml
diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml
new file mode 100644
index 00000000..8887f66d
--- /dev/null
+++ b/tests/data/benchmarks/plan/fake-plan.yaml
@@ -0,0 +1,10 @@
+name: fake plan
+description: fake benchmark plan for demonstration and testing
+config:
+ facility: local
+ engineer: local
+ driver: sample
+ collector: logfile
+ reporter: console
+QPIs:
+ - fake-qpi.yaml
diff --git a/tests/data/benchmarks/plans/verification.yaml b/tests/data/benchmarks/plans/verification.yaml
deleted file mode 100644
index b146ee37..00000000
--- a/tests/data/benchmarks/plans/verification.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-title: verification
-description: benchmark plan for QTIP verification
-info:
- facility: opnfv-ci
- engineer: opnfv-bot
-suites:
- - QPI_spec: compute.yaml
-# Uncomment next line to setup suite running condition
-# condition: {}
-# Uncomment the following lines setup case running configuration
-# cases:
-# - metric_spec: dhrystone.yaml
-# config: {}
-# - metric_spec: whetstone.yaml
-# config: {}
-# - metric_spec: ramspeed.yaml
-# config: {}
-# - metric_spec: dpi.yaml
-# config: {}
-# - metric_spec: ssl.yaml
-# config: {}
diff --git a/tests/unit/cli/test_module.py b/tests/unit/cli/test_module.py
deleted file mode 100644
index a8900684..00000000
--- a/tests/unit/cli/test_module.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import os
-import pytest
-import sys
-import yaml
-
-from click.testing import CliRunner
-from qtip.cli.entry import cli
-
-
-class TestClass(object):
- '''TODO(taseer) Remove hardcoded behaviour '''
-
- @pytest.fixture()
- def runner(self):
- return CliRunner()
-
- def test(self, runner):
- unit = 'perftest'
- test_file = 'data/helper/' + unit + '.yaml'
- path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, test_file)
-
- with open(path) as trial:
- content = trial.read()
- data = yaml.safe_load(content)['tests']
- if data is None:
- print("Unit Test does not exist")
- sys.exit(1)
- else:
- for i in range(0, len(data)):
- result = runner.invoke(cli, data[i]['command'])
- assert result.output == data[i]['output']
diff --git a/tests/unit/cli/test_options.py b/tests/unit/cli/test_options.py
new file mode 100644
index 00000000..5129b5cc
--- /dev/null
+++ b/tests/unit/cli/test_options.py
@@ -0,0 +1,32 @@
+###############################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+from click.testing import CliRunner
+
+from qtip.cli.entry import cli
+
+
+class TestClass(object):
+
+ @pytest.fixture()
+ def runner(self):
+ return CliRunner()
+
+ def test_verbose(self, runner):
+ result = runner.invoke(cli, ['-v'])
+ assert result.output == ''
+
+ def test_version(self, runner):
+ result = runner.invoke(cli, ['--version'])
+ assert '' in result.output
+
+ def test_debug(self, runner):
+ result = runner.invoke(cli, ['-d'])
+ assert '' in result.output
diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py
index 5eced700..d2be0388 100644
--- a/tests/unit/loader/metric_test.py
+++ b/tests/unit/loader/metric_test.py
@@ -9,8 +9,8 @@
import pytest
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
+from qtip.base.constant import PropName
+from qtip.loader.metric import MetricSpec
@pytest.fixture(scope='module')
@@ -29,17 +29,17 @@ def init_test(metric_spec):
def list_all_test():
metric_list = MetricSpec.list_all()
- assert len(list(metric_list)) is 1
+ assert len(list(metric_list)) is 6
for desc in metric_list:
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
+ assert PropName.NAME in desc
+ assert PropName.DESCRIPTION in desc
+ assert PropName.ABSPATH in desc
+ assert PropName.ABSPATH is not None
def content_test(metric):
- content = metric.content()
- assert Property.NAME in content
- assert Property.DESCRIPTION in content
- assert Property.WORKLOADS in content
- assert isinstance(content[Property.WORKLOADS], list)
+ content = metric.content
+ assert PropName.NAME in content
+ assert PropName.DESCRIPTION in content
+ assert PropName.WORKLOADS in content
+ assert isinstance(content[PropName.WORKLOADS], list)
diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py
index bfa1f580..c0d4b377 100644
--- a/tests/unit/loader/qpi_test.py
+++ b/tests/unit/loader/qpi_test.py
@@ -9,8 +9,8 @@
import pytest
-from qtip.base.benchmark import Algorithm, Property
-from qtip.spec.qpi import QPISpec
+from qtip.base.constant import AlgoName, PropName
+from qtip.loader.qpi import QPISpec
QPI_SPEC = 'compute.yaml'
@@ -31,22 +31,22 @@ def test_init(qpi_spec):
def test_list_all(benchmarks_root):
qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root])
- assert len(list(qpi_spec_list)) is 1
+ assert len(list(qpi_spec_list)) is 2
for item in qpi_spec_list:
- assert Property.NAME in item
- assert Property.CONTENT in item
- assert Property.ABSPATH in item
- assert Property.ABSPATH is not None
+ assert PropName.NAME in item
+ assert PropName.CONTENT in item
+ assert PropName.ABSPATH in item
+ assert PropName.ABSPATH is not None
def test_content(qpi_spec):
- content = qpi_spec.content()
- assert Property.DESCRIPTION in content
- assert Property.ALGORITHM in content
- assert Property.SECTIONS in content
+ content = qpi_spec.content
+ assert PropName.DESCRIPTION in content
+ assert PropName.ALGORITHM in content
+ assert PropName.SECTIONS in content
- assert content[Property.ALGORITHM] in Algorithm.__dict__.values()
- sections = content[Property.SECTIONS]
+ assert content[PropName.ALGORITHM] in AlgoName.__dict__.values()
+ sections = content[PropName.SECTIONS]
assert isinstance(sections, list)
for section in sections:
- assert Property.NAME in section
+ assert PropName.NAME in section
diff --git a/tests/unit/runner/case_test.py b/tests/unit/runner/case_test.py
deleted file mode 100644
index 59a54a84..00000000
--- a/tests/unit/runner/case_test.py
+++ /dev/null
@@ -1,15 +0,0 @@
-###############################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.spec.metric import MetricSpec
-
-
-def init_test(case):
- assert isinstance(case.metric_spec, MetricSpec)
- assert isinstance(case.config, dict)
diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py
deleted file mode 100644
index 6d14f7ae..00000000
--- a/tests/unit/runner/conftest.py
+++ /dev/null
@@ -1,30 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.base.benchmark import Property
-from qtip.runner.case import Case
-from qtip.runner.plan import Plan
-from qtip.runner.suite import Suite
-
-
-@pytest.fixture(scope='module')
-def plan(benchmarks_root):
- return Plan('verification.yaml', paths=[benchmarks_root])
-
-
-@pytest.fixture(scope='module')
-def suite(plan):
- return Suite(plan[Property.SUITES][0])
-
-
-@pytest.fixture(scope='module')
-def case(suite):
- return Case(suite[Property.CASES][0])
diff --git a/tests/unit/runner/plan_test.py b/tests/unit/runner/plan_test.py
index d783e5e9..7b3611d1 100644
--- a/tests/unit/runner/plan_test.py
+++ b/tests/unit/runner/plan_test.py
@@ -9,12 +9,12 @@
import pytest
-from qtip.base.benchmark import Property
+from qtip.base.constant import PropName
from qtip.runner.plan import Plan
def test_init(plan):
- assert plan.name == 'verification'
+ assert plan.name == 'fake plan'
with pytest.raises(TypeError) as excinfo:
Plan()
@@ -26,13 +26,13 @@ def test_list_all(benchmarks_root):
plan_list = Plan.list_all(paths=[benchmarks_root])
assert len(list(plan_list)) is 1
for desc in plan_list:
- assert Property.NAME in desc
- assert Property.CONTENT in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
+ assert PropName.NAME in desc
+ assert PropName.CONTENT in desc
+ assert PropName.ABSPATH in desc
+ assert PropName.ABSPATH is not None
def test_content(plan):
- content = plan.content()
- assert Property.TITLE in content
- assert Property.DESCRIPTION in content
+ content = plan.content
+ assert PropName.NAME in content
+ assert PropName.DESCRIPTION in content
diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py
deleted file mode 100644
index 7dad8f62..00000000
--- a/tests/unit/runner/suite_test.py
+++ /dev/null
@@ -1,27 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.runner.suite import Suite
-from qtip.runner.case import Case
-from qtip.spec.qpi import QPISpec
-
-
-def init_test(suite):
- assert isinstance(suite.qpi, QPISpec)
- assert isinstance(suite.condition, dict)
- assert isinstance(suite.cases, list)
- for case in suite.cases:
- assert isinstance(case, Case)
-
- with pytest.raises(TypeError) as excinfo:
- Suite()
- assert '__init__() takes exactly 2 arguments (1 given)' \
- in str(excinfo.value)