Implment https://wiki.opnfv.org/display/qtip/Design

Note that some obsolete test cases are marked expected failure, will
be deprecated after architecture evolution.

JIRA: QTIP-148
Change-Id: I52bc9391569d516e298d9e659517161b4dce794a
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>

5 files changed, 43 insertions, 94 deletions

diff --git a/qtip/runner/benchmark.py b/qtip/runner/benchmark.py
deleted file mode 100644
index 58e5a8d9..00000000
--- a/qtip/runner/benchmark.py
+++ /dev/null
@@ -1,52 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from itertools import chain
-from os import listdir
-from os import path
-
-
-class Property:
-    NAME = 'name'
-    DESCRIPTION = 'description'
-    ABSPATH = 'abspath'
-
-
-class Benchmark(object):
-    """Abstract class of QTIP benchmarks"""
-
-    _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
-                        'benchmarks')]
-
-    def __init__(self, name):
-        self.name = name
-        self._abspath = self._find(name)
-
-    def _find(self, name):
-        """find a benchmark in searching paths"""
-        for p in self._paths:
-            abspath = path.join(p, name)
-            if path.exists(abspath):
-                return abspath
-        return None
-
-    @classmethod
-    def list_all(cls):
-        """list all available benchmarks"""
-        names = chain.from_iterable([listdir(p) for p in cls._paths])
-        return [cls(name).describe() for name in names]
-
-    def describe(self):
-        """description of benchmark"""
-        # TODO(yujunz) read description from benchmark content
-        return {
-            Property.NAME: self.name,
-            Property.DESCRIPTION: 'QTIP benchmark',
-            Property.ABSPATH: self._abspath
-        }
diff --git a/qtip/runner/testplan.py b/qtip/runner/case.py
index f48a7147..eb3febc2 100644
--- a/qtip/runner/testplan.py
+++ b/qtip/runner/case.py
@@ -7,15 +7,11 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from os import path
+from qtip.base.benchmark import Property
+from qtip.spec.metric import MetricSpec
 
-from benchmark import Benchmark
 
-
-class TestPlan(Benchmark):
-    """WIP(yujunz):
-    a test plan is consist of test condition and several suites which can be
-    executed by user"""
-
-    # paths to search for suites
-    _paths = [path.join(p, 'testplan') for p in Benchmark._paths]
+class Case(object):
+    def __init__(self, spec, paths=None):
+        self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths)
+        self.config = spec[Property.CONFIG]
diff --git a/qtip/runner/perftest.py b/qtip/runner/perftest.py
deleted file mode 100644
index a9b54716..00000000
--- a/qtip/runner/perftest.py
+++ /dev/null
@@ -1,21 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from os import path
-
-from benchmark import Benchmark
-
-
-class PerfTest(Benchmark):
-    """WIP(yujunz):
-    a perftest is the driver of external performance test tools
-    It is usually referred in a suite to collect performance metric"""
-
-    # paths to search for perftest
-    _paths = [path.join(p, 'perftest') for p in Benchmark._paths]
diff --git a/qtip/runner/plan.py b/qtip/runner/plan.py
new file mode 100644
index 00000000..265ad8d7
--- /dev/null
+++ b/qtip/runner/plan.py
@@ -0,0 +1,26 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.benchmark import Benchmark, Property
+from qtip.runner.suite import Suite
+
+
+class Plan(Benchmark):
+    """
+    a benchmark plan is consist of basic information and  several suites"""
+
+    DEFAULT_DIR = 'plans'
+
+    def __init__(self, name, paths=None):
+        super(Plan, self).__init__(name, paths=paths)
+        content = self.content()
+
+        self.info = content[Property.INFO]
+        self.suites = [Suite(suite, paths=paths)
+                       for suite in content[Property.SUITES]]
diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py
index 0086a20e..55033d2b 100644
--- a/qtip/runner/suite.py
+++ b/qtip/runner/suite.py
@@ -7,16 +7,16 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from os import path
+from qtip.base.benchmark import Property
+from qtip.spec.qpi import QPISpec
+from qtip.runner.case import Case
 
-from benchmark import Benchmark
 
-
-class Suite(Benchmark):
-    """WIP(yujunz):
-    a suite is consist of one or several perf tests and produces one QPI.
-    It must be executed as part of testplan
-    """
-
-    # paths to search for suites
-    _paths = [path.join(p, 'suite') for p in Benchmark._paths]
+class Suite(object):
+    """a suite of benchmark cases under specified condition"""
+    def __init__(self, spec, paths=None):
+        self._paths = paths
+        self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths)
+        self.condition = spec.get(Property.CONDITION, {})
+        self.cases = [Case(case_spec, paths)
+                      for case_spec in spec.get(Property.CASES, [])]