Implment https://wiki.opnfv.org/display/qtip/Design

Note that some obsolete test cases are marked expected failure, will
be deprecated after architecture evolution.

JIRA: QTIP-148
Change-Id: I52bc9391569d516e298d9e659517161b4dce794a
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>

6 files changed, 95 insertions, 129 deletions

diff --git a/tests/unit/runner/case_test.py b/tests/unit/runner/case_test.py
new file mode 100644
index 00000000..59a54a84
--- /dev/null
+++ b/tests/unit/runner/case_test.py
@@ -0,0 +1,15 @@
+###############################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.spec.metric import MetricSpec
+
+
+def init_test(case):
+    assert isinstance(case.metric_spec, MetricSpec)
+    assert isinstance(case.config, dict)
diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py
new file mode 100644
index 00000000..6d14f7ae
--- /dev/null
+++ b/tests/unit/runner/conftest.py
@@ -0,0 +1,30 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+
+from qtip.base.benchmark import Property
+from qtip.runner.case import Case
+from qtip.runner.plan import Plan
+from qtip.runner.suite import Suite
+
+
+@pytest.fixture(scope='module')
+def plan(benchmarks_root):
+    return Plan('verification.yaml', paths=[benchmarks_root])
+
+
+@pytest.fixture(scope='module')
+def suite(plan):
+    return Suite(plan[Property.SUITES][0])
+
+
+@pytest.fixture(scope='module')
+def case(suite):
+    return Case(suite[Property.CASES][0])
diff --git a/tests/unit/runner/perftest_test.py b/tests/unit/runner/perftest_test.py
deleted file mode 100644
index 2b400ac8..00000000
--- a/tests/unit/runner/perftest_test.py
+++ /dev/null
@@ -1,48 +0,0 @@
-###############################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from os import path
-import pytest
-
-from qtip.runner.perftest import PerfTest
-from qtip.runner.benchmark import Property
-
-
-class TestPerfTestClass:
-    def test_attr(self):
-        assert len(PerfTest._paths) is 1
-
-
-class TestPerfTest:
-    PerfTest._paths = [path.join(path.dirname(__file__), path.pardir,
-                                 path.pardir, 'data', 'perftest')]
-
-    def test_init(self):
-        perftest = PerfTest('test-a')
-        assert perftest.name == 'test-a'
-
-        with pytest.raises(TypeError) as excinfo:
-            PerfTest()
-        assert '__init__() takes exactly 2 arguments (1 given)' \
-               in str(excinfo.value)
-
-    def test_list(self):
-        perftest_list = PerfTest.list_all()
-        assert len(list(perftest_list)) is 1
-        for desc in perftest_list:
-            assert Property.NAME in desc
-            assert Property.DESCRIPTION in desc
-            assert Property.ABSPATH in desc
-            assert Property.ABSPATH is not None
-
-    def test_describe(self):
-        desc = PerfTest('test-a').describe()
-        assert Property.NAME in desc
-        assert Property.DESCRIPTION in desc
-        assert Property.ABSPATH in desc
diff --git a/tests/unit/runner/plan_test.py b/tests/unit/runner/plan_test.py
new file mode 100644
index 00000000..d783e5e9
--- /dev/null
+++ b/tests/unit/runner/plan_test.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+
+from qtip.base.benchmark import Property
+from qtip.runner.plan import Plan
+
+
+def test_init(plan):
+    assert plan.name == 'verification'
+
+    with pytest.raises(TypeError) as excinfo:
+        Plan()
+    assert '__init__() takes at least 2 arguments (1 given)' \
+           in str(excinfo.value)
+
+
+def test_list_all(benchmarks_root):
+    plan_list = Plan.list_all(paths=[benchmarks_root])
+    assert len(list(plan_list)) is 1
+    for desc in plan_list:
+        assert Property.NAME in desc
+        assert Property.CONTENT in desc
+        assert Property.ABSPATH in desc
+        assert Property.ABSPATH is not None
+
+
+def test_content(plan):
+    content = plan.content()
+    assert Property.TITLE in content
+    assert Property.DESCRIPTION in content
diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py
index acfed82c..7dad8f62 100644
--- a/tests/unit/runner/suite_test.py
+++ b/tests/unit/runner/suite_test.py
@@ -7,42 +7,21 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from os import path
 import pytest
 
 from qtip.runner.suite import Suite
-from qtip.runner.benchmark import Property
+from qtip.runner.case import Case
+from qtip.spec.qpi import QPISpec
 
 
-class TestSuiteClass:
-    def test_attr(self):
-        assert len(Suite._paths) is 1
+def init_test(suite):
+    assert isinstance(suite.qpi, QPISpec)
+    assert isinstance(suite.condition, dict)
+    assert isinstance(suite.cases, list)
+    for case in suite.cases:
+        assert isinstance(case, Case)
 
-
-class TestSuite:
-    Suite._paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
-                              'data', 'suite')]
-
-    def test_init(self):
-        suite = Suite('suite-1')
-        assert suite.name == 'suite-1'
-
-        with pytest.raises(TypeError) as excinfo:
-            Suite()
-        assert '__init__() takes exactly 2 arguments (1 given)' \
-               in str(excinfo.value)
-
-    def test_list(self):
-        suite_list = Suite.list_all()
-        assert len(list(suite_list)) is 3
-        for suite_desc in suite_list:
-            assert Property.NAME in suite_desc
-            assert Property.DESCRIPTION in suite_desc
-            assert Property.ABSPATH in suite_desc
-            assert Property.ABSPATH is not None
-
-    def test_describe(self):
-        desc = Suite('suite-a').describe()
-        assert Property.NAME in desc
-        assert Property.DESCRIPTION in desc
-        assert Property.ABSPATH in desc
+    with pytest.raises(TypeError) as excinfo:
+        Suite()
+    assert '__init__() takes exactly 2 arguments (1 given)' \
+           in str(excinfo.value)
diff --git a/tests/unit/runner/testplan_test.py b/tests/unit/runner/testplan_test.py
deleted file mode 100644
index 7e42f557..00000000
--- a/tests/unit/runner/testplan_test.py
+++ /dev/null
@@ -1,48 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from os import path
-import pytest
-
-from qtip.runner.testplan import TestPlan
-from qtip.runner.benchmark import Property
-
-
-class TestTestPlanClass:
-    def test_attr(self):
-        assert len(TestPlan._paths) is 1
-
-
-class TestTestPlan:
-    TestPlan._paths = [path.join(path.dirname(__file__), path.pardir,
-                                 path.pardir, 'data', 'testplan')]
-
-    def test_init(self):
-        plan = TestPlan('plan-a')
-        assert plan.name == 'plan-a'
-
-        with pytest.raises(TypeError) as excinfo:
-            TestPlan()
-        assert '__init__() takes exactly 2 arguments (1 given)' \
-               in str(excinfo.value)
-
-    def test_list(self):
-        plan_list = TestPlan.list_all()
-        assert len(list(plan_list)) is 5
-        for desc in plan_list:
-            assert Property.NAME in desc
-            assert Property.DESCRIPTION in desc
-            assert Property.ABSPATH in desc
-            assert Property.ABSPATH is not None
-
-    def test_describe(self):
-        desc = TestPlan('plan-a').describe()
-        assert Property.NAME in desc
-        assert Property.DESCRIPTION in desc
-        assert Property.ABSPATH in desc