45 files changed, 365 insertions, 276 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 93c8bc9d..a3e56317 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -26,6 +26,7 @@ Active Reviewers
 
 Current list of active reviewers in gerrit group `qtip-reviewers`
 
+* Akhil Batra <akhil.batra@research.iiit.ac.in>
 * Serena Feng <feng.xiaowei@zte.com.cn>
 * Taseer Ahmed <taseer94@gmail.com>
 * Yujun Zhang <zhang.yujunz@zte.com.cn>
diff --git a/opt/servers/dev.yml b/opt/servers/dev.yml
deleted file mode 100644
index 2e0cca93..00000000
--- a/opt/servers/dev.yml
+++ /dev/null
@@ -1,10 +0,0 @@
----
-- hosts: development
-  roles:
-    - user
-    - docker
-    - elk
-    - mongo
-    - nginx
-    - qtip
-    - testapi
diff --git a/opt/servers/inventory b/opt/servers/inventory
index 0a862caa..1ba1dc45 100644
--- a/opt/servers/inventory
+++ b/opt/servers/inventory
@@ -1,8 +1,4 @@
-desk  ansible_host=desk.qtip.io   # 2vCPU/4GiB/16GiB (AWS us-east-1a)
 table ansible_host=table.qtip.io  # 1vCPU/1024MB/100Mbps/40GB (Aliyun Hongkong B)
 
-[development]
-desk
-
 [testing]
 table
diff --git a/opt/servers/test.yml b/opt/servers/test.yml
index 549b7255..ee3ded13 100644
--- a/opt/servers/test.yml
+++ b/opt/servers/test.yml
@@ -2,3 +2,9 @@
 - hosts: testing
   roles:
     - user
+    - docker
+    - elk
+    - mongo
+    - nginx
+    - qtip
+    - testapi
diff --git a/qtip/api/cmd/server.py b/qtip/api/cmd/server.py
index 852073a7..3b89af1e 100644
--- a/qtip/api/cmd/server.py
+++ b/qtip/api/cmd/server.py
@@ -13,7 +13,7 @@ from flask_restful_swagger import swagger
 import qtip.api.router.mapper as mapper
 
 app = Flask(__name__)
-api = swagger.docs(Api(app), apiVersion='0.1')
+api = swagger.docs(Api(app), apiVersion='0.1', description='QTIP API specs')
 
 
 def add_routers():
diff --git a/qtip/api/handler/job_handler.py b/qtip/api/handler/job_handler.py
index f230e596..eade9df7 100644
--- a/qtip/api/handler/job_handler.py
+++ b/qtip/api/handler/job_handler.py
@@ -1,8 +1,8 @@
 import threading
 from copy import copy
 
-from flask.ext.restful import Resource, reqparse
-from flask.ext.restful_swagger import swagger
+from flask_restful import Resource, reqparse
+from flask_restful_swagger import swagger
 from werkzeug.exceptions import abort
 
 from qtip.api.handler import db, result_handler
diff --git a/qtip/api/model/job_model.py b/qtip/api/model/job_model.py
index eef771b0..f7eb9fda 100644
--- a/qtip/api/model/job_model.py
+++ b/qtip/api/model/job_model.py
@@ -1,5 +1,5 @@
-from flask.ext.restful import fields
-from flask.ext.restful_swagger import swagger
+from flask_restful import fields
+from flask_restful_swagger import swagger
 
 
 @swagger.model
diff --git a/qtip/base/constant.py b/qtip/base/constant.py
new file mode 100644
index 00000000..187f0706
--- /dev/null
+++ b/qtip/base/constant.py
@@ -0,0 +1,50 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class AlgoName(object):
+    """algorithm names"""
+    ARITHMETIC_MEAN = 'arithmetic mean'
+    WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
+    GEOMETRIC_MEAN = 'geometric mean'
+    WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
+
+
+class PkgName(object):
+    """QTIP package names"""
+    COLLECTOR = 'collector'
+    DRIVER = 'driver'
+    REPORTER = 'reporter'
+    RUNNER = 'runner'
+    SPEC = 'spec'
+
+
+class PropName(object):
+    """property names"""
+    # list
+    NAME = 'name'
+    CONTENT = 'content'
+    ABSPATH = 'abspath'
+    # content
+    name = 'name'
+    DESCRIPTION = 'description'
+    # spec
+    SECTIONS = 'sections'
+    WEIGHT = 'weight'
+    ALGORITHM = 'algorithm'
+    METRICS = 'metrics'
+    WORKLOADS = 'workloads'
+    # plan
+    CONFIG = 'config'
+    FACILITY = 'facility'
+    ENGINEER = 'engineer'
+    DRIVER = 'driver'
+    COLLECTOR = 'collector'
+    REPORTER = 'reporter'
+    QPIS = 'QPIs'
diff --git a/qtip/base/error.py b/qtip/base/error.py
new file mode 100644
index 00000000..d364c532
--- /dev/null
+++ b/qtip/base/error.py
@@ -0,0 +1,23 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class QtipError(Exception):
+    pass
+
+
+class InvalidFormat(QtipError):
+    def __init__(self, filename):
+        self.filename = filename
+
+
+class NotFound(QtipError):
+    def __init__(self, module, package='qtip'):
+        self.package = package
+        self.module = module
diff --git a/qtip/cli/entry.py b/qtip/cli/entry.py
index c3ebf96a..0ecb06cd 100644
--- a/qtip/cli/entry.py
+++ b/qtip/cli/entry.py
@@ -7,13 +7,72 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+import os
+import sys
 import click
-from qtip.cli.commands import cmd_perftest
-from qtip.cli.commands import cmd_suite
+
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
-cli = click.CommandCollection(sources=[cmd_perftest.cli, cmd_suite.cli])
 
-if __name__ == '__main__':
-    cli()
+class Context(object):
+
+    ''' TODO(taseer) implementation'''
+
+    def __init__(self):
+        self.verbose = False
+        self.debug = False
+        self.version = 'qtip-1.0.0'
+
+    def log(self, msg, *args):
+        ''' Log message to stderr '''
+        pass
+
+    def verbose(self, msg, *args):
+        ''' Log message to stderr when verbose '''
+        pass
+
+    def version(self):
+        ''' Display programs version '''
+        pass
+
+    def debug(self, msg, *args):
+        ''' Log message to debug '''
+        pass
+
+
+pass_context = click.make_pass_decorator(Context, ensure=True)
+cmd_folder = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                          'commands'))
+
+
+class QtipCli(click.MultiCommand):
+
+    def list_commands(self, ctx):
+        rv = []
+        for filename in os.listdir(cmd_folder):
+            if filename.endswith('.py') and \
+               filename.startswith('cmd_'):
+                rv.append(filename[4:-3])
+        rv.sort()
+        return rv
+
+    def get_command(self, ctx, name):
+        try:
+            if sys.version_info[0] == 2:
+                name = name.encode('ascii', 'replace')
+            mod = __import__('qtip.cli.commands.cmd_' + name,
+                             None, None, ['cli'])
+        except ImportError:
+            return
+        return mod.cli
+
+
+@click.command(cls=QtipCli, context_settings=CONTEXT_SETTINGS,
+               invoke_without_command=True)
+@click.option('-v', '--verbose', is_flag=True, help='Enable verbose mode.')
+@click.option('-d', '--debug', is_flag=True, help='Enable debug mode.')
+@click.option('--version', is_flag=True, help='Shows program current version.')
+@pass_context
+def cli(ctx, verbose, version, debug):
+    pass
diff --git a/qtip/drivers/__init__.py b/qtip/collector/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/drivers/__init__.py
+++ b/qtip/collector/__init__.py
diff --git a/qtip/drivers/base.py b/qtip/collector/base.py
index 1aa8d8ad..cd8fc797 100644
--- a/qtip/drivers/base.py
+++ b/qtip/collector/base.py
@@ -8,5 +8,5 @@
 ##############################################################################
 
 
-class BaseDriver(object):
-    """performance testing tool driver"""
+class BaseCollector(object):
+    """performance metrics collector"""
diff --git a/qtip/agent/reporter.py b/qtip/collector/logfile.py
index b5c4acfa..6528ea9f 100644
--- a/qtip/agent/reporter.py
+++ b/qtip/collector/logfile.py
@@ -7,7 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+from base import BaseCollector
 
-class Reporter(object):
-    """generate test report and push test data to database"""
-    pass
+
+class LogfileCollector(BaseCollector):
+    """collect performance metrics from log files"""
diff --git a/qtip/spec/__init__.py b/qtip/driver/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/spec/__init__.py
+++ b/qtip/driver/__init__.py
diff --git a/qtip/drivers/ansible.py b/qtip/driver/ansible.py
index 04e9f9bd..04e9f9bd 100644
--- a/qtip/drivers/ansible.py
+++ b/qtip/driver/ansible.py
diff --git a/qtip/runner/case.py b/qtip/driver/base.py
index eb3febc2..6f5cab3c 100644
--- a/qtip/runner/case.py
+++ b/qtip/driver/base.py
@@ -7,11 +7,14 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
 
+class BaseDriver(object):
+    """performance testing tool driver"""
+    def pre_run(self):
+        pass
 
-class Case(object):
-    def __init__(self, spec, paths=None):
-        self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths)
-        self.config = spec[Property.CONFIG]
+    def run(self):
+        pass
+
+    def post_run(self):
+        pass
diff --git a/tests/unit/runner/case_test.py b/qtip/driver/sample.py
index 59a54a84..9b347949 100644
--- a/tests/unit/runner/case_test.py
+++ b/qtip/driver/sample.py
@@ -1,4 +1,4 @@
-###############################################################
+##############################################################################
 # Copyright (c) 2016 ZTE Corp and others.
 #
 # All rights reserved. This program and the accompanying materials
@@ -7,9 +7,9 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.spec.metric import MetricSpec
+
+from base import BaseDriver
 
 
-def init_test(case):
-    assert isinstance(case.metric_spec, MetricSpec)
-    assert isinstance(case.config, dict)
+class SampleDriver(BaseDriver):
+    """sample driver that generates random data for testing"""
diff --git a/qtip/drivers/yardstick.py b/qtip/driver/yardstick.py
index 83f1b3d8..83f1b3d8 100644
--- a/qtip/drivers/yardstick.py
+++ b/qtip/driver/yardstick.py
diff --git a/qtip/loader/__init__.py b/qtip/loader/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/loader/__init__.py
diff --git a/qtip/base/benchmark.py b/qtip/loader/base.py
index b38e6016..f7fcb669 100644
--- a/qtip/base/benchmark.py
+++ b/qtip/loader/base.py
@@ -12,52 +12,14 @@ from os import listdir
 from os import path
 import yaml
 
-
-class Property(object):
-    # list
-    NAME = 'name'
-    CONTENT = 'content'
-    ABSPATH = 'abspath'
-    # content
-    TITLE = 'title'
-    DESCRIPTION = 'description'
-    # spec
-    ALGORITHM = 'algorithm'
-    SECTIONS = 'sections'
-    WEIGHT = 'weight'
-    METRICS = 'metrics'
-    SPEC = 'spec'
-    WORKLOADS = 'workloads'
-    # plan
-    INFO = 'info'
-    FACILITY = 'facility'
-    ENGINEER = 'engineer'
-    SUITES = 'suites'
-    # suite
-    QPI_SPEC = 'QPI_spec'
-    CONDITION = 'condition'
-    CASES = 'cases'
-    # case
-    METRIC_SPEC = 'metric_spec'
-    CONFIG = 'config'
-
-
-class QtipError(Exception):
-    pass
-
-
-class Algorithm(object):
-    ARITHMETIC_MEAN = 'arithmetic mean'
-    WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
-    GEOMETRIC_MEAN = 'geometric mean'
-    WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
-
+from qtip.base.error import InvalidFormat, NotFound
+from qtip.base.constant import PropName
 
 ROOT_DIR = 'benchmarks'
 
 
-class Benchmark(object):
-    """Abstract class of QTIP benchmarks"""
+class BaseLoader(object):
+    """Abstract class of QTIP benchmark loader"""
     DEFAULT_DIR = '.'
     _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
                         ROOT_DIR)]
@@ -65,7 +27,16 @@ class Benchmark(object):
     def __init__(self, name, paths=None):
         self._file = name
         self._abspath = self._find(name, paths)
-        self.name = path.splitext(name)[0]
+
+        try:
+            content = yaml.safe_load(file(self._abspath))
+        except yaml.YAMLError:
+            # TODO(yujunz) log yaml error
+            raise InvalidFormat(self._abspath)
+
+        self.name = content[PropName.NAME] if PropName.NAME in content \
+            else path.splitext(name)[0]
+        self.content = content
 
     def _find(self, name, paths):
         """find a benchmark in searching paths"""
@@ -75,7 +46,7 @@ class Benchmark(object):
             abspath = path.join(p, name)
             if path.exists(abspath):
                 return abspath
-        raise QtipError("'{}' not found in paths: {}".format(name, paths))
+        raise NotFound(name, paths)
 
     @classmethod
     def list_all(cls, paths=None):
@@ -86,10 +57,6 @@ class Benchmark(object):
         for name in names:
             item = cls(name, paths=paths)
             yield {
-                Property.NAME: name,
-                Property.ABSPATH: item._abspath,
-                Property.CONTENT: item.content()}
-
-    def content(self):
-        """description of benchmark"""
-        return yaml.safe_load(file(self._abspath))
+                PropName.NAME: name,
+                PropName.ABSPATH: item._abspath,
+                PropName.CONTENT: item.content}
diff --git a/qtip/spec/metric.py b/qtip/loader/metric.py
index e9c70547..d6174e8f 100644
--- a/qtip/spec/metric.py
+++ b/qtip/loader/metric.py
@@ -7,10 +7,10 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.base.benchmark import Benchmark
+from base import BaseLoader
 
 
-class MetricSpec(Benchmark):
+class MetricSpec(BaseLoader):
     """metrics in QTIP are categorized by performance test tools, such as
     dhrystone, whetstone and etc"""
-    DEFAULT_DIR = 'metrics'
+    DEFAULT_DIR = 'metric'
diff --git a/qtip/spec/qpi.py b/qtip/loader/qpi.py
index b7d7aa02..2972cbd7 100644
--- a/qtip/spec/qpi.py
+++ b/qtip/loader/qpi.py
@@ -7,10 +7,10 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.base.benchmark import Benchmark
+from base import BaseLoader
 
 
-class QPISpec(Benchmark):
+class QPISpec(BaseLoader):
     """
     a QPI specification defines how to calculate a performance index from
      collected metrics.
diff --git a/qtip/reporter/__init__.py b/qtip/reporter/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/reporter/__init__.py
diff --git a/qtip/reporter/base.py b/qtip/reporter/base.py
new file mode 100644
index 00000000..b931d14d
--- /dev/null
+++ b/qtip/reporter/base.py
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseReporter(object):
+    """benchmark result reporter"""
+    def __init__(self, collector=None):
+        self.collector = collector
diff --git a/qtip/agent/collector.py b/qtip/reporter/console.py
index 3df21379..136ce358 100644
--- a/qtip/agent/collector.py
+++ b/qtip/reporter/console.py
@@ -7,7 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+from base import BaseReporter
 
-class Collector(object):
-    """collect test result and test condition"""
-    pass
+
+class ConsoleReporter(BaseReporter):
+    """report result to console"""
diff --git a/qtip/runner/base.py b/qtip/runner/base.py
new file mode 100644
index 00000000..07fba104
--- /dev/null
+++ b/qtip/runner/base.py
@@ -0,0 +1,45 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.constant import PkgName, PropName
+from qtip.base.error import NotFound
+from qtip.collector.logfile import LogfileCollector
+from qtip.driver.sample import SampleDriver
+from qtip.loader.base import BaseLoader
+from qtip.reporter.console import ConsoleReporter
+
+
+class BaseRunner(BaseLoader):
+    def __init__(self, name, paths=None, config=None):
+        super(BaseRunner, self).__init__(name, paths=paths)
+        if config is None:
+            config = self.content[PropName.CONFIG]
+
+        driver_name = config[PropName.DRIVER]
+        collector_name = config[PropName.COLLECTOR]
+        reporter_name = config[PropName.REPORTER]
+
+        # TODO(yujunz) dynamically load modules by name
+
+        if driver_name == 'sample':
+            self.driver = SampleDriver()
+        else:
+            raise NotFound(driver_name, package=PkgName.DRIVER)
+
+        if collector_name == 'logfile':
+            self.collector = LogfileCollector()
+        else:
+            raise NotFound(collector_name,
+                           package=PkgName.COLLECTOR)
+
+        if reporter_name == 'console':
+            self.reporter = ConsoleReporter()
+        else:
+            raise NotFound(reporter_name,
+                           package=PkgName.REPORTER)
diff --git a/qtip/runner/plan.py b/qtip/runner/plan.py
index 265ad8d7..f6c1c3bb 100644
--- a/qtip/runner/plan.py
+++ b/qtip/runner/plan.py
@@ -7,20 +7,19 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.base.benchmark import Benchmark, Property
-from qtip.runner.suite import Suite
+from qtip.base.constant import PropName
+from qtip.runner.base import BaseRunner
+from qtip.loader.qpi import QPISpec
 
 
-class Plan(Benchmark):
+class Plan(BaseRunner):
+    """
+    a benchmark plan is consist of configuration and a QPI list
     """
-    a benchmark plan is consist of basic information and  several suites"""
 
-    DEFAULT_DIR = 'plans'
+    DEFAULT_DIR = 'plan'
 
     def __init__(self, name, paths=None):
-        super(Plan, self).__init__(name, paths=paths)
-        content = self.content()
-
-        self.info = content[Property.INFO]
-        self.suites = [Suite(suite, paths=paths)
-                       for suite in content[Property.SUITES]]
+        super(Plan, self).__init__(name, paths)
+        self.qpis = [QPISpec(qpi, paths=paths)
+                     for qpi in self.content[PropName.QPIS]]
diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py
deleted file mode 100644
index 55033d2b..00000000
--- a/qtip/runner/suite.py
+++ /dev/null
@@ -1,22 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.benchmark import Property
-from qtip.spec.qpi import QPISpec
-from qtip.runner.case import Case
-
-
-class Suite(object):
-    """a suite of benchmark cases under specified condition"""
-    def __init__(self, spec, paths=None):
-        self._paths = paths
-        self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths)
-        self.condition = spec.get(Property.CONDITION, {})
-        self.cases = [Case(case_spec, paths)
-                      for case_spec in spec.get(Property.CASES, [])]
diff --git a/tests/conftest.py b/tests/conftest.py
index 8e2ecf7c..f1ec91fb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -10,6 +10,8 @@
 from os import path
 import pytest
 
+from qtip.runner.plan import Plan
+
 
 @pytest.fixture(scope='session')
 def data_root():
@@ -19,3 +21,8 @@ def data_root():
 @pytest.fixture(scope='session')
 def benchmarks_root(data_root):
     return path.join(data_root, 'benchmarks')
+
+
+@pytest.fixture(scope='session')
+def plan(benchmarks_root):
+    return Plan('fake-plan.yaml', [benchmarks_root])
diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml
new file mode 100644
index 00000000..d75c6568
--- /dev/null
+++ b/tests/data/benchmarks/QPI/fake-qpi.yaml
@@ -0,0 +1,9 @@
+name: Fake QPI
+description: a fake QPI producing random result
+algorithm: weighted arithmetic mean
+sections:
+- name: Fake Section
+  weight: 0.5
+  algorithm: geometric mean
+  metrics:
+    - fake_metric.yaml
diff --git a/tests/data/benchmarks/metrics/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml
index b0d55ed2..b0d55ed2 100644
--- a/tests/data/benchmarks/metrics/dhrystone.yaml
+++ b/tests/data/benchmarks/metric/dhrystone.yaml
diff --git a/tests/data/benchmarks/metrics/dpi.yaml b/tests/data/benchmarks/metric/dpi.yaml
index fc24c8d1..fc24c8d1 100644
--- a/tests/data/benchmarks/metrics/dpi.yaml
+++ b/tests/data/benchmarks/metric/dpi.yaml
diff --git a/tests/data/benchmarks/metric/fake-metric.yaml b/tests/data/benchmarks/metric/fake-metric.yaml
new file mode 100644
index 00000000..c5fcc5c2
--- /dev/null
+++ b/tests/data/benchmarks/metric/fake-metric.yaml
@@ -0,0 +1,7 @@
+name: fake compute
+description: >
+  a fake compute performance index which generate random result
+workloads:  # all supported workloads
+  - fake_workload_1
+  - fake_workload_2
+  - fake_workload_3
diff --git a/tests/data/benchmarks/metrics/ramspeed.yaml b/tests/data/benchmarks/metric/ramspeed.yaml
index bb7618b7..bb7618b7 100644
--- a/tests/data/benchmarks/metrics/ramspeed.yaml
+++ b/tests/data/benchmarks/metric/ramspeed.yaml
diff --git a/tests/data/benchmarks/metrics/ssl.yaml b/tests/data/benchmarks/metric/ssl.yaml
index 21e8add7..21e8add7 100644
--- a/tests/data/benchmarks/metrics/ssl.yaml
+++ b/tests/data/benchmarks/metric/ssl.yaml
diff --git a/tests/data/benchmarks/metrics/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml
index d83680c4..d83680c4 100644
--- a/tests/data/benchmarks/metrics/whetstone.yaml
+++ b/tests/data/benchmarks/metric/whetstone.yaml
diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml
new file mode 100644
index 00000000..8887f66d
--- /dev/null
+++ b/tests/data/benchmarks/plan/fake-plan.yaml
@@ -0,0 +1,10 @@
+name: fake plan
+description: fake benchmark plan for demonstration and testing
+config:
+  facility: local
+  engineer: local
+  driver: sample
+  collector: logfile
+  reporter: console
+QPIs:
+  - fake-qpi.yaml
diff --git a/tests/data/benchmarks/plans/verification.yaml b/tests/data/benchmarks/plans/verification.yaml
deleted file mode 100644
index b146ee37..00000000
--- a/tests/data/benchmarks/plans/verification.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-title: verification
-description: benchmark plan for QTIP verification
-info:
-  facility: opnfv-ci
-  engineer: opnfv-bot
-suites:
-  - QPI_spec: compute.yaml
-# Uncomment next line to setup suite running condition
-#    condition: {}
-# Uncomment the following lines setup case running configuration
-#    cases:
-#      - metric_spec: dhrystone.yaml
-#        config: {}
-#      - metric_spec: whetstone.yaml
-#        config: {}
-#      - metric_spec: ramspeed.yaml
-#        config: {}
-#      - metric_spec: dpi.yaml
-#        config: {}
-#      - metric_spec: ssl.yaml
-#        config: {}
diff --git a/tests/unit/cli/test_module.py b/tests/unit/cli/test_module.py
deleted file mode 100644
index a8900684..00000000
--- a/tests/unit/cli/test_module.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import os
-import pytest
-import sys
-import yaml
-
-from click.testing import CliRunner
-from qtip.cli.entry import cli
-
-
-class TestClass(object):
-    '''TODO(taseer) Remove hardcoded behaviour '''
-
-    @pytest.fixture()
-    def runner(self):
-        return CliRunner()
-
-    def test(self, runner):
-        unit = 'perftest'
-        test_file = 'data/helper/' + unit + '.yaml'
-        path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, test_file)
-
-        with open(path) as trial:
-            content = trial.read()
-            data = yaml.safe_load(content)['tests']
-            if data is None:
-                print("Unit Test does not exist")
-                sys.exit(1)
-            else:
-                for i in range(0, len(data)):
-                    result = runner.invoke(cli, data[i]['command'])
-                    assert result.output == data[i]['output']
diff --git a/tests/unit/cli/test_options.py b/tests/unit/cli/test_options.py
new file mode 100644
index 00000000..5129b5cc
--- /dev/null
+++ b/tests/unit/cli/test_options.py
@@ -0,0 +1,32 @@
+###############################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+from click.testing import CliRunner
+
+from qtip.cli.entry import cli
+
+
+class TestClass(object):
+
+    @pytest.fixture()
+    def runner(self):
+        return CliRunner()
+
+    def test_verbose(self, runner):
+        result = runner.invoke(cli, ['-v'])
+        assert result.output == ''
+
+    def test_version(self, runner):
+        result = runner.invoke(cli, ['--version'])
+        assert '' in result.output
+
+    def test_debug(self, runner):
+        result = runner.invoke(cli, ['-d'])
+        assert '' in result.output
diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py
index 5eced700..d2be0388 100644
--- a/tests/unit/loader/metric_test.py
+++ b/tests/unit/loader/metric_test.py
@@ -9,8 +9,8 @@
 
 import pytest
 
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
+from qtip.base.constant import PropName
+from qtip.loader.metric import MetricSpec
 
 
 @pytest.fixture(scope='module')
@@ -29,17 +29,17 @@ def init_test(metric_spec):
 
 def list_all_test():
     metric_list = MetricSpec.list_all()
-    assert len(list(metric_list)) is 1
+    assert len(list(metric_list)) is 6
     for desc in metric_list:
-        assert Property.NAME in desc
-        assert Property.DESCRIPTION in desc
-        assert Property.ABSPATH in desc
-        assert Property.ABSPATH is not None
+        assert PropName.NAME in desc
+        assert PropName.DESCRIPTION in desc
+        assert PropName.ABSPATH in desc
+        assert PropName.ABSPATH is not None
 
 
 def content_test(metric):
-    content = metric.content()
-    assert Property.NAME in content
-    assert Property.DESCRIPTION in content
-    assert Property.WORKLOADS in content
-    assert isinstance(content[Property.WORKLOADS], list)
+    content = metric.content
+    assert PropName.NAME in content
+    assert PropName.DESCRIPTION in content
+    assert PropName.WORKLOADS in content
+    assert isinstance(content[PropName.WORKLOADS], list)
diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py
index bfa1f580..c0d4b377 100644
--- a/tests/unit/loader/qpi_test.py
+++ b/tests/unit/loader/qpi_test.py
@@ -9,8 +9,8 @@
 
 import pytest
 
-from qtip.base.benchmark import Algorithm, Property
-from qtip.spec.qpi import QPISpec
+from qtip.base.constant import AlgoName, PropName
+from qtip.loader.qpi import QPISpec
 
 QPI_SPEC = 'compute.yaml'
 
@@ -31,22 +31,22 @@ def test_init(qpi_spec):
 
 def test_list_all(benchmarks_root):
     qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root])
-    assert len(list(qpi_spec_list)) is 1
+    assert len(list(qpi_spec_list)) is 2
     for item in qpi_spec_list:
-        assert Property.NAME in item
-        assert Property.CONTENT in item
-        assert Property.ABSPATH in item
-        assert Property.ABSPATH is not None
+        assert PropName.NAME in item
+        assert PropName.CONTENT in item
+        assert PropName.ABSPATH in item
+        assert PropName.ABSPATH is not None
 
 
 def test_content(qpi_spec):
-    content = qpi_spec.content()
-    assert Property.DESCRIPTION in content
-    assert Property.ALGORITHM in content
-    assert Property.SECTIONS in content
+    content = qpi_spec.content
+    assert PropName.DESCRIPTION in content
+    assert PropName.ALGORITHM in content
+    assert PropName.SECTIONS in content
 
-    assert content[Property.ALGORITHM] in Algorithm.__dict__.values()
-    sections = content[Property.SECTIONS]
+    assert content[PropName.ALGORITHM] in AlgoName.__dict__.values()
+    sections = content[PropName.SECTIONS]
     assert isinstance(sections, list)
     for section in sections:
-        assert Property.NAME in section
+        assert PropName.NAME in section
diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py
deleted file mode 100644
index 6d14f7ae..00000000
--- a/tests/unit/runner/conftest.py
+++ /dev/null
@@ -1,30 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.base.benchmark import Property
-from qtip.runner.case import Case
-from qtip.runner.plan import Plan
-from qtip.runner.suite import Suite
-
-
-@pytest.fixture(scope='module')
-def plan(benchmarks_root):
-    return Plan('verification.yaml', paths=[benchmarks_root])
-
-
-@pytest.fixture(scope='module')
-def suite(plan):
-    return Suite(plan[Property.SUITES][0])
-
-
-@pytest.fixture(scope='module')
-def case(suite):
-    return Case(suite[Property.CASES][0])
diff --git a/tests/unit/runner/plan_test.py b/tests/unit/runner/plan_test.py
index d783e5e9..7b3611d1 100644
--- a/tests/unit/runner/plan_test.py
+++ b/tests/unit/runner/plan_test.py
@@ -9,12 +9,12 @@
 
 import pytest
 
-from qtip.base.benchmark import Property
+from qtip.base.constant import PropName
 from qtip.runner.plan import Plan
 
 
 def test_init(plan):
-    assert plan.name == 'verification'
+    assert plan.name == 'fake plan'
 
     with pytest.raises(TypeError) as excinfo:
         Plan()
@@ -26,13 +26,13 @@ def test_list_all(benchmarks_root):
     plan_list = Plan.list_all(paths=[benchmarks_root])
     assert len(list(plan_list)) is 1
     for desc in plan_list:
-        assert Property.NAME in desc
-        assert Property.CONTENT in desc
-        assert Property.ABSPATH in desc
-        assert Property.ABSPATH is not None
+        assert PropName.NAME in desc
+        assert PropName.CONTENT in desc
+        assert PropName.ABSPATH in desc
+        assert PropName.ABSPATH is not None
 
 
 def test_content(plan):
-    content = plan.content()
-    assert Property.TITLE in content
-    assert Property.DESCRIPTION in content
+    content = plan.content
+    assert PropName.NAME in content
+    assert PropName.DESCRIPTION in content
diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py
deleted file mode 100644
index 7dad8f62..00000000
--- a/tests/unit/runner/suite_test.py
+++ /dev/null
@@ -1,27 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.runner.suite import Suite
-from qtip.runner.case import Case
-from qtip.spec.qpi import QPISpec
-
-
-def init_test(suite):
-    assert isinstance(suite.qpi, QPISpec)
-    assert isinstance(suite.condition, dict)
-    assert isinstance(suite.cases, list)
-    for case in suite.cases:
-        assert isinstance(case, Case)
-
-    with pytest.raises(TypeError) as excinfo:
-        Suite()
-    assert '__init__() takes exactly 2 arguments (1 given)' \
-           in str(excinfo.value)