217 files changed, 1275 insertions, 410 deletions

diff --git a/.gitignore b/.gitignore
index aab79ed0..7c7892db 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,3 +66,10 @@ target/
 /docs_build/
 /docs_output/
 .idea
+
+# setup.py cache
+AUTHORS
+ChangeLog
+
+# unignore sample logfiles
+!/opt/plugin/doctor/sample/*/*
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 93c8bc9d..a4929172 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -13,19 +13,31 @@ Peer review is the most important communication channel between developers.
 Every subtle change to the code or document **MUST** be reviewed before
 submission.
 
-Add group `qtip-reviewers` in [gerrit][gr] when you consider a patch set is ready.
+Add group `qtip-reviewers` in [gerrit][gr] when you consider a patch set is
+ready.
 
-Please make sure there is at least one `+1` or `+2` from others before
-submitting a patch set.
 
-Note: only members in `ldap/opnfv-gerrit-qtip-submitters` have permission
-to submit. The current members are listed in [INFO][if].
+Rule for Submit
+---------------
+
+Only members in `ldap/opnfv-gerrit-qtip-submitters`, i.e. committers have
+permission to submit. The current committers are listed in [INFO][if].
+
+Before submitting a patch set, please make sure one of the following criteria is
+met:
+
+* **Normal**: at least one `+2` from others
+* **Fast Track**: no objection from other committers for one or more working day
+
+NOTE: we introduce **fast track** because currently there are not enough active
+committers reviewing the patch sets.
 
 Active Reviewers
 ----------------
 
 Current list of active reviewers in gerrit group `qtip-reviewers`
 
+* Akhil Batra <akhil.batra@research.iiit.ac.in>
 * Serena Feng <feng.xiaowei@zte.com.cn>
 * Taseer Ahmed <taseer94@gmail.com>
 * Yujun Zhang <zhang.yujunz@zte.com.cn>
diff --git a/DEVELOP.md b/DEVELOP.md
new file mode 100644
index 00000000..9ec3a93d
--- /dev/null
+++ b/DEVELOP.md
@@ -0,0 +1,101 @@
+# QTIP Developer Guide
+
+This guide is about how to **develop** QTIP.
+
+If you just want to use it for performance benchmark, check the user guide
+instead.
+
+## Getting Started
+
+### Source code
+
+```bash
+~$ git clone https://gerrit.opnfv.org/gerrit/qtip
+```
+
+### VirtualEnv
+
+It is recommended to use [virtualenv](https://virtualenv.pypa.io) to isolate
+your development environment from system, especially when you are working on
+several different projects.
+
+### Testing
+
+QTIP use [tox](https://tox.readthedocs.io) to automate the testing tasks
+
+```bash
+$ pip install tox
+$ tox
+```
+
+## Architecture
+
+**TODO**: move to design spec
+
+QTIP has a flexible architecture to allow different deployment mode
+
+- **Standalone**: full feature performance benchmark platform.
+- **Agent**: minimal agent driven by external test runners.
+
+### Standalone Mode (Solo)
+
+QTIP instance deployed in container, VM or host generate benchmark report and
+push data to Indices Hub for storage and visualization.
+
+![solo](https://wiki.opnfv.org/download/attachments/8687017/Standalone.png?api=v2)
+
+### Agent Mode (Melody)
+
+QTIP Collector and Reporter driven by external test framework or runner such as
+[yardstick](https://wiki.opnfv.org/display/yardstick),
+[pytest](http://doc.pytest.org/) and etc.
+
+![melody](https://wiki.opnfv.org/download/attachments/8687017/Agent.png?api=v2)
+
+## Core Modules
+
+TBD
+
+- loader
+- runner
+- collector
+- reporter
+
+## Drivers
+
+TBD
+
+- ansible
+- yardstick
+
+## Interfaces
+
+### Agent
+
+TBD
+
+### CLI
+
+TBD
+
+### API
+
+TBD
+
+## Assets
+
+**TODO**: move to user guide
+
+- benchmark plan
+- QPI spec
+- metric spec
+
+## Docker Image
+
+TBD
+
+## Annex
+
+### Directories
+
+TBD
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 369fa6c8..f2a543e6 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,8 +2,7 @@
 #####Docker container for QTIP############
 ##########################################
 
-
-FROM ubuntu:14.04
+FROM ubuntu:16.04
 MAINTAINER Yujun Zhang <zhang.yujunz@zte.com.cn>
 LABEL version="0.1" description="OPNFV QTIP Docker container"
 
@@ -17,31 +16,31 @@ WORKDIR /home/opnfv
 
 # Packaged Dependencies
 RUN apt-get update && apt-get install -y \
-software-properties-common \
-wget \
-git \
-gcc \
-ssh \
-expect \
-python-matplotlib \
-python-dev \
-python-pip \
-libjpeg62 \
-libjpeg62-dev \
-zlib1g-dev \
-python-tk \
-curl \
-supervisor \
-python-setuptools \
---no-install-recommends
-
+    software-properties-common \
+    wget \
+    git \
+    gcc \
+    ssh \
+    expect \
+    python-matplotlib \
+    python-dev \
+    python-pip \
+    libjpeg62 \
+    libjpeg62-dev \
+    zlib1g-dev \
+    python-tk \
+    curl \
+    supervisor \
+    python-setuptools \
+    --no-install-recommends \
+&& rm -rf /var/lib/apt/lists/*
+
+RUN pip install -U pip
 RUN pip install 'setuptools>=17.1'
 
 RUN apt-add-repository ppa:ansible/ansible -y
 RUN apt-key update -y
-RUN apt-get update -y
-RUN apt-get install ansible --force-yes -y
-
+RUN apt-get update && apt-get install ansible -y
 
 RUN mkdir -p ${REPOS_DIR}
 RUN mkdir -p /root/.ssh
@@ -54,7 +53,6 @@ RUN chmod 700 /root/.ssh
 COPY ansible.cfg.default /root/.ansible.cfg
 
 #Cloning Repos
-
 RUN git config --global http.sslVerify false
 RUN git clone -b $BRANCH https://gerrit.opnfv.org/gerrit/qtip $REPOS_DIR/qtip
 RUN git clone https://gerrit.opnfv.org/gerrit/releng $REPOS_DIR/releng
@@ -62,7 +60,6 @@ RUN git clone https://gerrit.opnfv.org/gerrit/releng $REPOS_DIR/releng
 RUN pip install -r $REPOS_DIR/qtip/requirements.txt
 
 #Config supervisor
-
 RUN mkdir -p /var/log/supervisor
 RUN locale-gen en_US en_US.UTF-8
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
diff --git a/docs/designspec/index.rst b/docs/designspec/index.rst
index d3bed217..b6dd0c01 100644
--- a/docs/designspec/index.rst
+++ b/docs/designspec/index.rst
@@ -12,3 +12,4 @@ QTIP Design Specifications
 
    dashboard.rst
    compute_QPI.rst
+   integration_with_yardstick.rst
diff --git a/docs/designspec/integration_with_yardstick.rst b/docs/designspec/integration_with_yardstick.rst
new file mode 100644
index 00000000..a8298d6f
--- /dev/null
+++ b/docs/designspec/integration_with_yardstick.rst
@@ -0,0 +1,92 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) 2016 ZTE Corp.
+
+
+**************************
+Integration with Yardstick
+**************************
+
+Problem description
+===================
+
+For each specified QPI [1]_, QTIP needs to select a suite of test cases and collect
+required test results. Based on these results, Qtip calculates the score.
+
+Proposed change
+===============
+Qtip has a flexible architecture [2]_ to support different mode: standalone and agent.
+It is recommended to use **agent mode** to work with existing test runners. Yardstick will
+act as a runner to generate test result and trigger Qtip agent on the completion of test.
+
+
+Work Items in Yardstick
+-----------------------
+
+1. Create a customized suite in Yardstick
+
+Yardstick not only has many existing suites but also support customized suites. Qtip could
+create a suite named **Qtip-PoC** in Yardstick repo to verify workflow of Qtip agent mode.
+
+2. Launch Qtip in Yardstick
+
+Whether to launch Qtip will be determined by checking the existence of OS environment
+variable *QTIP*. If it exists, Qtip will be launched by using Yardstick CLI
+`yardstick plugin install` [3]_.
+
+3. Yardstick interacts with Qtip
+
+See
+`Yardstick-Qtip+integration <https://wiki.opnfv.org/display/yardstick/Yardstick-Qtip+integration>`_
+for details.
+
+Work Items in Qtip
+------------------
+
+1. Provide an API for Yardstick to post test result and environment info
+
+After completing test execution, Yardstick will post test result and enviroment info with
+JSON format via QTIP API. See
+`Yardstick-Qtip+integration <https://wiki.opnfv.org/display/yardstick/Yardstick-Qtip+integration>`_
+for details.
+
+2. Parse yardstick test result
+
+When Qtip agent receive Yarstick test result and enviroment info, Qtip agent will extract
+metrics which is definded in metric spec configuration file. Based on these metrics, Qtip
+agent will caculate QPI.
+
+3. Provide an API for querying QPI
+
+Qtip will provide an API for querying QPI. See
+`Yardstick-Qtip+integration <https://wiki.opnfv.org/display/yardstick/Yardstick-Qtip+integration>`_
+for details.
+
+Implementation
+==============
+
+Assignee(s)
+-----------
+
+*Primary assignee:*
+  wu.zhihui
+
+*Other contributors*
+  TBD
+
+Testing
+=======
+
+The changes will be covered by new unit test.
+
+Documentation
+=============
+
+TBD
+
+Reference
+=========
+
+.. [1] QTIP performance index
+.. [2] https://wiki.opnfv.org/display/qtip/Architecture
+.. [3] https://wiki.opnfv.org/display/yardstick/How+to+install+a+plug-in+into+Yardstick
diff --git a/legacy/DO-NOT-DELETE b/legacy/DO-NOT-DELETE
new file mode 100644
index 00000000..fdecaad1
--- /dev/null
+++ b/legacy/DO-NOT-DELETE
@@ -0,0 +1,2 @@
+The legacy code is no longer maintained. But they should be kept until we finish
+migration to new architecture.
diff --git a/qtip/agent/__init__.py b/legacy/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/agent/__init__.py
+++ b/legacy/__init__.py
diff --git a/qtip/api/handler/__init__.py b/legacy/api/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/api/handler/__init__.py
+++ b/legacy/api/__init__.py
diff --git a/qtip/api/model/__init__.py b/legacy/api/handler/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/api/model/__init__.py
+++ b/legacy/api/handler/__init__.py
diff --git a/qtip/api/handler/db.py b/legacy/api/handler/db.py
index 24fc27a5..24fc27a5 100644
--- a/qtip/api/handler/db.py
+++ b/legacy/api/handler/db.py
diff --git a/qtip/api/handler/job_handler.py b/legacy/api/handler/job_handler.py
index eade9df7..b75da5ff 100644
--- a/qtip/api/handler/job_handler.py
+++ b/legacy/api/handler/job_handler.py
@@ -3,11 +3,11 @@ from copy import copy
 
 from flask_restful import Resource, reqparse
 from flask_restful_swagger import swagger
-from werkzeug.exceptions import abort
-
-from qtip.api.handler import db, result_handler
 from qtip.api.model.job_model import JobResponseModel
 from qtip.utils import args_handler as args_handler
+from werkzeug.exceptions import abort
+
+from legacy.api.handler import db, result_handler
 
 
 class Job(Resource):
diff --git a/qtip/api/handler/result_handler.py b/legacy/api/handler/result_handler.py
index 3d1d592e..3d1d592e 100644
--- a/qtip/api/handler/result_handler.py
+++ b/legacy/api/handler/result_handler.py
diff --git a/qtip/api/router/__init__.py b/legacy/api/model/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/api/router/__init__.py
+++ b/legacy/api/model/__init__.py
diff --git a/qtip/api/model/job_model.py b/legacy/api/model/job_model.py
index f7eb9fda..f7eb9fda 100644
--- a/qtip/api/model/job_model.py
+++ b/legacy/api/model/job_model.py
diff --git a/qtip/drivers/__init__.py b/legacy/api/router/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/drivers/__init__.py
+++ b/legacy/api/router/__init__.py
diff --git a/qtip/api/router/mapper.py b/legacy/api/router/mapper.py
index a5f029ac..8839f3f5 100644
--- a/qtip/api/router/mapper.py
+++ b/legacy/api/router/mapper.py
@@ -1,4 +1,4 @@
-from qtip.api.handler.job_handler import Job, JobList
+from legacy.api.handler import Job, JobList
 
 
 mappers = [
diff --git a/benchmarks/perftest/common/git_proxy_pbook.yaml b/legacy/assets/perftest/common/git_proxy_pbook.yaml
index 5cb6f450..5cb6f450 100644
--- a/benchmarks/perftest/common/git_proxy_pbook.yaml
+++ b/legacy/assets/perftest/common/git_proxy_pbook.yaml
diff --git a/benchmarks/perftest/common/sys_info_pbook.yaml b/legacy/assets/perftest/common/sys_info_pbook.yaml
index 5c2d8f79..5c2d8f79 100644
--- a/benchmarks/perftest/common/sys_info_pbook.yaml
+++ b/legacy/assets/perftest/common/sys_info_pbook.yaml
diff --git a/benchmarks/perftest/common/sys_proxy_pbook.yaml b/legacy/assets/perftest/common/sys_proxy_pbook.yaml
index bf4a8ccb..bf4a8ccb 100644
--- a/benchmarks/perftest/common/sys_proxy_pbook.yaml
+++ b/legacy/assets/perftest/common/sys_proxy_pbook.yaml
diff --git a/benchmarks/perftest/dhrystone.yaml b/legacy/assets/perftest/dhrystone.yaml
index 896aadfc..896aadfc 100644
--- a/benchmarks/perftest/dhrystone.yaml
+++ b/legacy/assets/perftest/dhrystone.yaml
diff --git a/benchmarks/perftest/dpi.yaml b/legacy/assets/perftest/dpi.yaml
index 2a10bef6..2a10bef6 100644
--- a/benchmarks/perftest/dpi.yaml
+++ b/legacy/assets/perftest/dpi.yaml
diff --git a/benchmarks/perftest/etc/dpi_average.sh b/legacy/assets/perftest/etc/dpi_average.sh
index 405d3ff6..405d3ff6 100644
--- a/benchmarks/perftest/etc/dpi_average.sh
+++ b/legacy/assets/perftest/etc/dpi_average.sh
diff --git a/benchmarks/perftest/etc/fio_test_job b/legacy/assets/perftest/etc/fio_test_job
index 6817abca..6817abca 100644
--- a/benchmarks/perftest/etc/fio_test_job
+++ b/legacy/assets/perftest/etc/fio_test_job
diff --git a/benchmarks/perftest/etc/info_collect.py b/legacy/assets/perftest/etc/info_collect.py
index 3fc35d5a..3fc35d5a 100644
--- a/benchmarks/perftest/etc/info_collect.py
+++ b/legacy/assets/perftest/etc/info_collect.py
diff --git a/benchmarks/perftest/fio.yaml b/legacy/assets/perftest/fio.yaml
index 0da9407d..0da9407d 100644
--- a/benchmarks/perftest/fio.yaml
+++ b/legacy/assets/perftest/fio.yaml
diff --git a/benchmarks/perftest/iperf.yaml b/legacy/assets/perftest/iperf.yaml
index a1755f7c..a1755f7c 100644
--- a/benchmarks/perftest/iperf.yaml
+++ b/legacy/assets/perftest/iperf.yaml
diff --git a/benchmarks/perftest/ramspeed.yaml b/legacy/assets/perftest/ramspeed.yaml
index 83ecd8bf..83ecd8bf 100644
--- a/benchmarks/perftest/ramspeed.yaml
+++ b/legacy/assets/perftest/ramspeed.yaml
diff --git a/benchmarks/perftest/ssl.yaml b/legacy/assets/perftest/ssl.yaml
index 6002bfff..6002bfff 100644
--- a/benchmarks/perftest/ssl.yaml
+++ b/legacy/assets/perftest/ssl.yaml
diff --git a/benchmarks/perftest/summary b/legacy/assets/perftest/summary
index 5891408c..5891408c 100644
--- a/benchmarks/perftest/summary
+++ b/legacy/assets/perftest/summary
diff --git a/benchmarks/perftest/whetstone.yaml b/legacy/assets/perftest/whetstone.yaml
index d6eae85f..d6eae85f 100644
--- a/benchmarks/perftest/whetstone.yaml
+++ b/legacy/assets/perftest/whetstone.yaml
diff --git a/benchmarks/suite/compute b/legacy/assets/suite/compute
index 3bf1b184..3bf1b184 100644
--- a/benchmarks/suite/compute
+++ b/legacy/assets/suite/compute
diff --git a/benchmarks/suite/compute.yaml b/legacy/assets/suite/compute.yaml
index 0761a87b..197d5720 100644
--- a/benchmarks/suite/compute.yaml
+++ b/legacy/assets/suite/compute.yaml
@@ -1,12 +1,12 @@
 QPI: compute
 description: sample performance index of computing
 
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
 
 section:
 - name: Integer
   weight: 0.3
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: dhrystone
       workloads:
@@ -14,7 +14,7 @@ section:
         - multi_cpu
 - name: Floating
   weight: 0.3
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: whetstone
       workloads:
@@ -22,7 +22,7 @@ section:
         - multi_cpu
 - name: Memory
   weight: 0.2
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: ramspeed
       workloads:
@@ -30,7 +30,7 @@ section:
           - float: [add, average, copy, scale, triad]
 - name: DPI
   weight: 0.1
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: dpi
       workloads:
@@ -38,7 +38,7 @@ section:
         - pps
 - name: SSL
   weight: 0.1
-  algorithm: geometric mean
+  formula: geometric mean
   perftests:
     - name: ssl
       workloads:
diff --git a/benchmarks/suite/network b/legacy/assets/suite/network
index 58ce5cb9..58ce5cb9 100644
--- a/benchmarks/suite/network
+++ b/legacy/assets/suite/network
diff --git a/benchmarks/suite/storage b/legacy/assets/suite/storage
index f3068dd5..f3068dd5 100644
--- a/benchmarks/suite/storage
+++ b/legacy/assets/suite/storage
diff --git a/benchmarks/testplan/default/compute/dhrystone_bm.yaml b/legacy/assets/testplan/default/compute/dhrystone_bm.yaml
index 64741537..64741537 100644
--- a/benchmarks/testplan/default/compute/dhrystone_bm.yaml
+++ b/legacy/assets/testplan/default/compute/dhrystone_bm.yaml
diff --git a/benchmarks/testplan/default/compute/dhrystone_vm.yaml b/legacy/assets/testplan/default/compute/dhrystone_vm.yaml
index 8f5a4165..8f5a4165 100644
--- a/benchmarks/testplan/default/compute/dhrystone_vm.yaml
+++ b/legacy/assets/testplan/default/compute/dhrystone_vm.yaml
diff --git a/benchmarks/testplan/default/compute/dpi_bm.yaml b/legacy/assets/testplan/default/compute/dpi_bm.yaml
index 14cb6d0d..14cb6d0d 100644
--- a/benchmarks/testplan/default/compute/dpi_bm.yaml
+++ b/legacy/assets/testplan/default/compute/dpi_bm.yaml
diff --git a/benchmarks/testplan/default/compute/dpi_vm.yaml b/legacy/assets/testplan/default/compute/dpi_vm.yaml
index 10e86993..10e86993 100644
--- a/benchmarks/testplan/default/compute/dpi_vm.yaml
+++ b/legacy/assets/testplan/default/compute/dpi_vm.yaml
diff --git a/benchmarks/testplan/default/compute/ramspeed_bm.yaml b/legacy/assets/testplan/default/compute/ramspeed_bm.yaml
index ed8fc700..ed8fc700 100644
--- a/benchmarks/testplan/default/compute/ramspeed_bm.yaml
+++ b/legacy/assets/testplan/default/compute/ramspeed_bm.yaml
diff --git a/benchmarks/testplan/default/compute/ramspeed_vm.yaml b/legacy/assets/testplan/default/compute/ramspeed_vm.yaml
index a6a4363f..a6a4363f 100644
--- a/benchmarks/testplan/default/compute/ramspeed_vm.yaml
+++ b/legacy/assets/testplan/default/compute/ramspeed_vm.yaml
diff --git a/benchmarks/testplan/default/compute/ssl_bm.yaml b/legacy/assets/testplan/default/compute/ssl_bm.yaml
index cc5a8903..cc5a8903 100644
--- a/benchmarks/testplan/default/compute/ssl_bm.yaml
+++ b/legacy/assets/testplan/default/compute/ssl_bm.yaml
diff --git a/benchmarks/testplan/default/compute/ssl_vm.yaml b/legacy/assets/testplan/default/compute/ssl_vm.yaml
index 37824896..37824896 100644
--- a/benchmarks/testplan/default/compute/ssl_vm.yaml
+++ b/legacy/assets/testplan/default/compute/ssl_vm.yaml
diff --git a/benchmarks/testplan/default/compute/whetstone_bm.yaml b/legacy/assets/testplan/default/compute/whetstone_bm.yaml
index 3c128b60..3c128b60 100644
--- a/benchmarks/testplan/default/compute/whetstone_bm.yaml
+++ b/legacy/assets/testplan/default/compute/whetstone_bm.yaml
diff --git a/benchmarks/testplan/default/compute/whetstone_vm.yaml b/legacy/assets/testplan/default/compute/whetstone_vm.yaml
index 0f1e8748..0f1e8748 100644
--- a/benchmarks/testplan/default/compute/whetstone_vm.yaml
+++ b/legacy/assets/testplan/default/compute/whetstone_vm.yaml
diff --git a/benchmarks/testplan/default/network/iperf_bm.yaml b/legacy/assets/testplan/default/network/iperf_bm.yaml
index 3aa8310d..3aa8310d 100644
--- a/benchmarks/testplan/default/network/iperf_bm.yaml
+++ b/legacy/assets/testplan/default/network/iperf_bm.yaml
diff --git a/benchmarks/testplan/default/network/iperf_vm.yaml b/legacy/assets/testplan/default/network/iperf_vm.yaml
index 49bf13ad..49bf13ad 100644
--- a/benchmarks/testplan/default/network/iperf_vm.yaml
+++ b/legacy/assets/testplan/default/network/iperf_vm.yaml
diff --git a/benchmarks/testplan/default/network/iperf_vm_2.yaml b/legacy/assets/testplan/default/network/iperf_vm_2.yaml
index c5b94715..c5b94715 100644
--- a/benchmarks/testplan/default/network/iperf_vm_2.yaml
+++ b/legacy/assets/testplan/default/network/iperf_vm_2.yaml
diff --git a/benchmarks/testplan/default/storage/fio_bm.yaml b/legacy/assets/testplan/default/storage/fio_bm.yaml
index d0001ea2..d0001ea2 100644
--- a/benchmarks/testplan/default/storage/fio_bm.yaml
+++ b/legacy/assets/testplan/default/storage/fio_bm.yaml
diff --git a/benchmarks/testplan/default/storage/fio_vm.yaml b/legacy/assets/testplan/default/storage/fio_vm.yaml
index 0e7f121b..0e7f121b 100644
--- a/benchmarks/testplan/default/storage/fio_vm.yaml
+++ b/legacy/assets/testplan/default/storage/fio_vm.yaml
diff --git a/qtip/cli/helper.py b/legacy/cli/helper.py
index acfecf8d..acfecf8d 100644
--- a/qtip/cli/helper.py
+++ b/legacy/cli/helper.py
diff --git a/config/SampleHeat.yaml b/legacy/config/SampleHeat.yaml
index a42cdb79..a42cdb79 100644
--- a/config/SampleHeat.yaml
+++ b/legacy/config/SampleHeat.yaml
diff --git a/tests/data/hosts b/legacy/data/hosts
index 0a0ac539..0a0ac539 100644
--- a/tests/data/hosts
+++ b/legacy/data/hosts
diff --git a/legacy/data/my_key.pem b/legacy/data/my_key.pem
new file mode 100644
index 00000000..d7c96f24
--- /dev/null
+++ b/legacy/data/my_key.pem
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAntmA9ybqcxQKr9R3iTbNr+89ZJwlt5+gLbT8VR9sUAYCEEJn
+xX7DX5djpSdQ1OoxJun/HE0ByKPXCIqGq3sHnxQ/3Wh80UGlyiSXgS8/p8NlfgPr
+DIDuVNhKJlsobsfTVXL789i512rqf2zFBWfoesFgZee6ACrSYN3hdNICFOwtbmHA
+g+xEs00yGmbcFDuBQnDeR2yPpV6G4AtrU5zwliVj7fMzrB5w9De20ydbFqxjPdOD
+gbfwrhQQs82pv7vfJCFByjsSlwP4mcznKgOt/aO2y/B1ZvL+dOsCi8D3H6Ggrg7R
+wmiIRaCijYX4SycYxdn+RkZpp9g0AyR/potcKwIDAQABAoIBADncgF2Gj1/brQjf
+G6ufiszLGFHNju+T9YSwqDlZeNqtVZMWnTYTNpdbTbCa4Zast7q1AXgNlNjA1VMH
+IobUCbKobZr4tH6Eqx82tPkZfNZfFlkQlE25qRa+skPGcLPpldFKUPxBtXACQeio
+dhvB/ay5Q6PbBKWK85bVO4qR9LApxHCDQgovGtasUF1/wu5z4vOCP3KLtlE268m+
+XJ/4IZX2erBOfxOGPqp+qDK8FTP1NcuGFEkhUa8Tr406CIXptTDyPc+bMZANIGDM
+hhww6VR5aHxsHWgu1UITVZuDh5fJ8U6n2utqTm/QLT25DUPKoTJSAjLet7fnm4Cw
+hee7GgECgYEAziiWq4peA9WrJYTlR8bNRLgsMjZx/AphyJQpFCQsgCMuB9wIluTT
+5Di4jvdvKclsFTh6foxnltltX6O8tL+vrzfbK9fBB3A2T1V1sDMOznhI47Mmoqk6
+9le6aMmpTIhNFJMu1DpsuoxRLIJaNkWLCwWaIE1ZggqCgbjIdMOJt20CgYEAxUDo
+DUf0zi/qYZVVTnziubcqu0kjUWhmeEiv08WFst8l2fAwHBILNms5ot4TY5MlFxta
+vLsWaXC7o6tqofSHF52cD8AbUWEnvK2NtCA7gW+0xBtQEPqrtItA0YjblFXw7qwb
+QUValSo535bFYJjp5foJxTZXg9rErF4iUgExqvcCgYBK5t5PNvePhxsmh5FSMb20
+oQGVwMhLHW6HlKUUJ0xvxdhcjKomQL9npOAROX8O/JqjXyDKR7L/UMH3RKM0PIzV
+KYMc+8erLy7cRh17RiG0DGnXKOj5omjExfz5Q4OaRc9TkWIrQ4rrgD2h4T5rh3rM
+J13nLJM/txfRj9Rs7/piRQKBgCm3RFwqr5c7gvIIRvZGaxyjpCHfodyBm8osdSLw
+Two5LyQcK9CD8GMd4h6ToL4aTGnKmzH2zxKViNlgUzjVIgzYhDzAgAaB6Yl5mtJR
+TsRc/3nJ3PD6Un3oRVkK/IUud7nCJDF1nWaWe47RsARx0mWUr6RJjdCQ368kaVW2
+cu6ZAoGBALD4Gw+AsH6/za7gmRIqlzBURHpHCgKutQKv3UbD1hVc5tDAoYwBjjod
+AVY8N8+AmX/nmJOVcch/dAnICkLrZn5Bm/q52/3xCAlnGkwNHyW4G99lZCfNEQrx
+sVAZ4FNrtMC9Xwtj0o73ojqKP4gxVkljbybnbuyNSXZo14EbXuJU
+-----END RSA PRIVATE KEY-----
diff --git a/tests/data/output/hosts b/legacy/data/output/hosts
index 9b47df0e..9b47df0e 100644
--- a/tests/data/output/hosts
+++ b/legacy/data/output/hosts
diff --git a/tests/data/schema/test_bm_schema.yaml b/legacy/data/schema/test_bm_schema.yaml
index a7c27e3f..a7c27e3f 100644
--- a/tests/data/schema/test_bm_schema.yaml
+++ b/legacy/data/schema/test_bm_schema.yaml
diff --git a/tests/data/schema/test_vm_schema.yaml b/legacy/data/schema/test_vm_schema.yaml
index 524f8fe4..524f8fe4 100644
--- a/tests/data/schema/test_vm_schema.yaml
+++ b/legacy/data/schema/test_vm_schema.yaml
diff --git a/tests/data/test.retry b/legacy/data/test.retry
index 7b9ad531..7b9ad531 100644
--- a/tests/data/test.retry
+++ b/legacy/data/test.retry
diff --git a/tests/data/test.yml b/legacy/data/test.yml
index 270e86fd..270e86fd 100644
--- a/tests/data/test.yml
+++ b/legacy/data/test.yml
diff --git a/tests/data/testplan/bm_ping.yaml b/legacy/data/testplan/bm_ping.yaml
index 41d696e2..41d696e2 100644
--- a/tests/data/testplan/bm_ping.yaml
+++ b/legacy/data/testplan/bm_ping.yaml
diff --git a/tests/data/testplan/bm_with_proxy.yaml b/legacy/data/testplan/bm_with_proxy.yaml
index 1d73300b..1d73300b 100644
--- a/tests/data/testplan/bm_with_proxy.yaml
+++ b/legacy/data/testplan/bm_with_proxy.yaml
diff --git a/tests/data/testplan/bm_without_proxy.yaml b/legacy/data/testplan/bm_without_proxy.yaml
index a9ae3b71..a9ae3b71 100644
--- a/tests/data/testplan/bm_without_proxy.yaml
+++ b/legacy/data/testplan/bm_without_proxy.yaml
diff --git a/tests/data/testplan/vm.yaml b/legacy/data/testplan/vm.yaml
index 4c8453ca..4c8453ca 100644
--- a/tests/data/testplan/vm.yaml
+++ b/legacy/data/testplan/vm.yaml
diff --git a/tests/data/testplan/vm_error.yaml b/legacy/data/testplan/vm_error.yaml
index f13d3a00..f13d3a00 100644
--- a/tests/data/testplan/vm_error.yaml
+++ b/legacy/data/testplan/vm_error.yaml
diff --git a/docker/README.md b/legacy/docker/README.md
index 35ac0935..35ac0935 100644
--- a/docker/README.md
+++ b/legacy/docker/README.md
diff --git a/docker/cleanup_qtip_image.sh b/legacy/docker/cleanup_qtip_image.sh
index 9c2b59db..9c2b59db 100644
--- a/docker/cleanup_qtip_image.sh
+++ b/legacy/docker/cleanup_qtip_image.sh
diff --git a/docker/prepare_qtip_image.sh b/legacy/docker/prepare_qtip_image.sh
index 4095c806..4095c806 100644
--- a/docker/prepare_qtip_image.sh
+++ b/legacy/docker/prepare_qtip_image.sh
diff --git a/docker/push_db.sh b/legacy/docker/push_db.sh
index 50341eac..50341eac 100755
--- a/docker/push_db.sh
+++ b/legacy/docker/push_db.sh
diff --git a/docker/run_qtip.sh b/legacy/docker/run_qtip.sh
index 98abf139..98abf139 100755
--- a/docker/run_qtip.sh
+++ b/legacy/docker/run_qtip.sh
diff --git a/qtip/run.py b/legacy/run.py
index a2c26eda..a2c26eda 100644
--- a/qtip/run.py
+++ b/legacy/run.py
diff --git a/qtip/spec/__init__.py b/legacy/scripts/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/spec/__init__.py
+++ b/legacy/scripts/__init__.py
diff --git a/scripts/cleanup_creds.sh b/legacy/scripts/cleanup_creds.sh
index b4eee924..b4eee924 100755
--- a/scripts/cleanup_creds.sh
+++ b/legacy/scripts/cleanup_creds.sh
diff --git a/scripts/fetch_compute_ips.sh b/legacy/scripts/fetch_compute_ips.sh
index 4bdc9a48..4bdc9a48 100755
--- a/scripts/fetch_compute_ips.sh
+++ b/legacy/scripts/fetch_compute_ips.sh
diff --git a/scripts/get_env_info.sh b/legacy/scripts/get_env_info.sh
index cd49ac87..cd49ac87 100755
--- a/scripts/get_env_info.sh
+++ b/legacy/scripts/get_env_info.sh
diff --git a/scripts/qtip_creds.sh b/legacy/scripts/qtip_creds.sh
index af051ac5..af051ac5 100755
--- a/scripts/qtip_creds.sh
+++ b/legacy/scripts/qtip_creds.sh
diff --git a/qtip/utils/__init__.py b/legacy/scripts/ref_results/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/utils/__init__.py
+++ b/legacy/scripts/ref_results/__init__.py
diff --git a/scripts/ref_results/compute_benchmarks_indices.py b/legacy/scripts/ref_results/compute_benchmarks_indices.py
index 0b6eae36..0b6eae36 100644
--- a/scripts/ref_results/compute_benchmarks_indices.py
+++ b/legacy/scripts/ref_results/compute_benchmarks_indices.py
diff --git a/scripts/ref_results/index_calculation.py b/legacy/scripts/ref_results/index_calculation.py
index 95c3c4a6..95c3c4a6 100644
--- a/scripts/ref_results/index_calculation.py
+++ b/legacy/scripts/ref_results/index_calculation.py
diff --git a/scripts/ref_results/network_benchmarks_indices.py b/legacy/scripts/ref_results/network_benchmarks_indices.py
index c19d18eb..c19d18eb 100644
--- a/scripts/ref_results/network_benchmarks_indices.py
+++ b/legacy/scripts/ref_results/network_benchmarks_indices.py
diff --git a/scripts/ref_results/reference.json b/legacy/scripts/ref_results/reference.json
index cfcbfc3b..cfcbfc3b 100644
--- a/scripts/ref_results/reference.json
+++ b/legacy/scripts/ref_results/reference.json
diff --git a/scripts/ref_results/result_accum.py b/legacy/scripts/ref_results/result_accum.py
index 6cd55886..6cd55886 100644
--- a/scripts/ref_results/result_accum.py
+++ b/legacy/scripts/ref_results/result_accum.py
diff --git a/scripts/ref_results/storage_benchmarks_indices.py b/legacy/scripts/ref_results/storage_benchmarks_indices.py
index a5aef638..a5aef638 100644
--- a/scripts/ref_results/storage_benchmarks_indices.py
+++ b/legacy/scripts/ref_results/storage_benchmarks_indices.py
diff --git a/scripts/ref_results/suite_result.py b/legacy/scripts/ref_results/suite_result.py
index 66213391..66213391 100644
--- a/scripts/ref_results/suite_result.py
+++ b/legacy/scripts/ref_results/suite_result.py
diff --git a/scripts/ssh_exch.exp b/legacy/scripts/ssh_exch.exp
index c52140b7..c52140b7 100644
--- a/scripts/ssh_exch.exp
+++ b/legacy/scripts/ssh_exch.exp
diff --git a/qtip/utils/dashboard/__init__.py b/legacy/tests/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/utils/dashboard/__init__.py
+++ b/legacy/tests/__init__.py
diff --git a/tests/unit/utils/ansible_api_test.py b/legacy/tests/ansible_api_test.py
index 6f286fc3..6f286fc3 100644
--- a/tests/unit/utils/ansible_api_test.py
+++ b/legacy/tests/ansible_api_test.py
diff --git a/qtip/utils/report/__init__.py b/legacy/tests/api/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/utils/report/__init__.py
+++ b/legacy/tests/api/__init__.py
diff --git a/tests/unit/api/test_server.py b/legacy/tests/api/test_server.py
index e9364d3d..e9364d3d 100644
--- a/tests/unit/api/test_server.py
+++ b/legacy/tests/api/test_server.py
diff --git a/tests/unit/utils/args_handler_test.py b/legacy/tests/args_handler_test.py
index dceca1f5..dceca1f5 100644
--- a/tests/unit/utils/args_handler_test.py
+++ b/legacy/tests/args_handler_test.py
diff --git a/tests/unit/utils/cli_test.py b/legacy/tests/cli_test.py
index 0f3e4158..0f3e4158 100644
--- a/tests/unit/utils/cli_test.py
+++ b/legacy/tests/cli_test.py
diff --git a/tests/unit/utils/create_zones_test.py b/legacy/tests/create_zones_test.py
index dcfff5ec..dcfff5ec 100644
--- a/tests/unit/utils/create_zones_test.py
+++ b/legacy/tests/create_zones_test.py
diff --git a/tests/unit/utils/driver_test.py b/legacy/tests/driver_test.py
index 432ce1ae..432ce1ae 100644
--- a/tests/unit/utils/driver_test.py
+++ b/legacy/tests/driver_test.py
diff --git a/tests/unit/utils/env_setup_test.py b/legacy/tests/env_setup_test.py
index dea48190..dea48190 100644
--- a/tests/unit/utils/env_setup_test.py
+++ b/legacy/tests/env_setup_test.py
diff --git a/qtip/utils/transform/__init__.py b/legacy/tests/functional/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/utils/transform/__init__.py
+++ b/legacy/tests/functional/__init__.py
diff --git a/tests/functional/yaml_schema_test.py b/legacy/tests/functional/yaml_schema_test.py
index a975dca6..a975dca6 100644
--- a/tests/functional/yaml_schema_test.py
+++ b/legacy/tests/functional/yaml_schema_test.py
diff --git a/tests/data/helper/perftest.yaml b/legacy/tests/helper/perftest.yaml
index 26c58452..26c58452 100644
--- a/tests/data/helper/perftest.yaml
+++ b/legacy/tests/helper/perftest.yaml
diff --git a/tests/data/helper/suite.yaml b/legacy/tests/helper/suite.yaml
index 718ae440..718ae440 100644
--- a/tests/data/helper/suite.yaml
+++ b/legacy/tests/helper/suite.yaml
diff --git a/tests/data/helper/version.yaml b/legacy/tests/helper/version.yaml
index b23f16f2..b23f16f2 100644
--- a/tests/data/helper/version.yaml
+++ b/legacy/tests/helper/version.yaml
diff --git a/tests/unit/utils/spawn_vm_test.py b/legacy/tests/spawn_vm_test.py
index ba237378..ba237378 100644
--- a/tests/unit/utils/spawn_vm_test.py
+++ b/legacy/tests/spawn_vm_test.py
diff --git a/scripts/__init__.py b/legacy/utils/__init__.py
index e69de29b..e69de29b 100644
--- a/scripts/__init__.py
+++ b/legacy/utils/__init__.py
diff --git a/qtip/utils/ansible_api.py b/legacy/utils/ansible_api.py
index 9e1d249e..9e1d249e 100644
--- a/qtip/utils/ansible_api.py
+++ b/legacy/utils/ansible_api.py
diff --git a/qtip/utils/args_handler.py b/legacy/utils/args_handler.py
index 993b1035..993b1035 100644
--- a/qtip/utils/args_handler.py
+++ b/legacy/utils/args_handler.py
diff --git a/qtip/utils/cli.py b/legacy/utils/cli.py
index 5e566f27..5e566f27 100644
--- a/qtip/utils/cli.py
+++ b/legacy/utils/cli.py
diff --git a/qtip/utils/create_zones.py b/legacy/utils/create_zones.py
index 5e378c83..5e378c83 100644
--- a/qtip/utils/create_zones.py
+++ b/legacy/utils/create_zones.py
diff --git a/scripts/ref_results/__init__.py b/legacy/utils/dashboard/__init__.py
index e69de29b..e69de29b 100644
--- a/scripts/ref_results/__init__.py
+++ b/legacy/utils/dashboard/__init__.py
diff --git a/qtip/utils/dashboard/pushtoDB.py b/legacy/utils/dashboard/pushtoDB.py
index 427d39c4..427d39c4 100644
--- a/qtip/utils/dashboard/pushtoDB.py
+++ b/legacy/utils/dashboard/pushtoDB.py
diff --git a/qtip/utils/driver.py b/legacy/utils/driver.py
index 9894e0f5..9894e0f5 100644
--- a/qtip/utils/driver.py
+++ b/legacy/utils/driver.py
diff --git a/qtip/utils/env_setup.py b/legacy/utils/env_setup.py
index 7bbedfcf..7bbedfcf 100644
--- a/qtip/utils/env_setup.py
+++ b/legacy/utils/env_setup.py
diff --git a/tests/functional/__init__.py b/legacy/utils/report/__init__.py
index e69de29b..e69de29b 100644
--- a/tests/functional/__init__.py
+++ b/legacy/utils/report/__init__.py
diff --git a/qtip/utils/report/get_indices.py b/legacy/utils/report/get_indices.py
index 91219c0b..91219c0b 100644
--- a/qtip/utils/report/get_indices.py
+++ b/legacy/utils/report/get_indices.py
diff --git a/qtip/utils/report/get_results.py b/legacy/utils/report/get_results.py
index 23fd5383..23fd5383 100644
--- a/qtip/utils/report/get_results.py
+++ b/legacy/utils/report/get_results.py
diff --git a/qtip/utils/report/qtip_graph.py b/legacy/utils/report/qtip_graph.py
index acbda40c..acbda40c 100644
--- a/qtip/utils/report/qtip_graph.py
+++ b/legacy/utils/report/qtip_graph.py
diff --git a/qtip/utils/report/qtip_report.py b/legacy/utils/report/qtip_report.py
index 6809e892..6809e892 100644
--- a/qtip/utils/report/qtip_report.py
+++ b/legacy/utils/report/qtip_report.py
diff --git a/qtip/utils/spawn_vm.py b/legacy/utils/spawn_vm.py
index f38c9a3a..f38c9a3a 100644
--- a/qtip/utils/spawn_vm.py
+++ b/legacy/utils/spawn_vm.py
diff --git a/tests/unit/api/__init__.py b/legacy/utils/transform/__init__.py
index e69de29b..e69de29b 100644
--- a/tests/unit/api/__init__.py
+++ b/legacy/utils/transform/__init__.py
diff --git a/qtip/utils/transform/dpi_transform.py b/legacy/utils/transform/dpi_transform.py
index ee29d8e2..ee29d8e2 100644
--- a/qtip/utils/transform/dpi_transform.py
+++ b/legacy/utils/transform/dpi_transform.py
diff --git a/qtip/utils/transform/final_report.py b/legacy/utils/transform/final_report.py
index 274742d4..274742d4 100644
--- a/qtip/utils/transform/final_report.py
+++ b/legacy/utils/transform/final_report.py
diff --git a/qtip/utils/transform/fio_transform.py b/legacy/utils/transform/fio_transform.py
index 5ecac823..5ecac823 100755
--- a/qtip/utils/transform/fio_transform.py
+++ b/legacy/utils/transform/fio_transform.py
diff --git a/qtip/utils/transform/iperf_transform.py b/legacy/utils/transform/iperf_transform.py
index b52e4634..b52e4634 100644
--- a/qtip/utils/transform/iperf_transform.py
+++ b/legacy/utils/transform/iperf_transform.py
diff --git a/qtip/utils/transform/ramspeed_transform.py b/legacy/utils/transform/ramspeed_transform.py
index 960f84fc..960f84fc 100644
--- a/qtip/utils/transform/ramspeed_transform.py
+++ b/legacy/utils/transform/ramspeed_transform.py
diff --git a/qtip/utils/transform/ssl_transform.py b/legacy/utils/transform/ssl_transform.py
index de84d24b..de84d24b 100644
--- a/qtip/utils/transform/ssl_transform.py
+++ b/legacy/utils/transform/ssl_transform.py
diff --git a/qtip/utils/transform/ubench_transform.py b/legacy/utils/transform/ubench_transform.py
index ab5fe171..ab5fe171 100644
--- a/qtip/utils/transform/ubench_transform.py
+++ b/legacy/utils/transform/ubench_transform.py
diff --git a/tests/unit/utils/__init__.py b/opt/__init__.py
index e69de29b..e69de29b 100644
--- a/tests/unit/utils/__init__.py
+++ b/opt/__init__.py
diff --git a/opt/servers/ansible.cfg b/opt/infra/ansible.cfg
index 93022c4e..93022c4e 100644
--- a/opt/servers/ansible.cfg
+++ b/opt/infra/ansible.cfg
diff --git a/opt/servers/inventory b/opt/infra/inventory
index 1ba1dc45..1ba1dc45 100644
--- a/opt/servers/inventory
+++ b/opt/infra/inventory
diff --git a/opt/servers/roles/docker/handlers/main.yml b/opt/infra/roles/docker/handlers/main.yml
index 985f8d41..985f8d41 100644
--- a/opt/servers/roles/docker/handlers/main.yml
+++ b/opt/infra/roles/docker/handlers/main.yml
diff --git a/opt/servers/roles/docker/tasks/main.yml b/opt/infra/roles/docker/tasks/main.yml
index 1cf3bea0..1cf3bea0 100644
--- a/opt/servers/roles/docker/tasks/main.yml
+++ b/opt/infra/roles/docker/tasks/main.yml
diff --git a/opt/servers/roles/elk/tasks/main.yml b/opt/infra/roles/elk/tasks/main.yml
index ed987f1a..ed987f1a 100644
--- a/opt/servers/roles/elk/tasks/main.yml
+++ b/opt/infra/roles/elk/tasks/main.yml
diff --git a/opt/servers/roles/mongo/tasks/main.yml b/opt/infra/roles/mongo/tasks/main.yml
index 81fb49b5..81fb49b5 100644
--- a/opt/servers/roles/mongo/tasks/main.yml
+++ b/opt/infra/roles/mongo/tasks/main.yml
diff --git a/opt/servers/roles/nginx/defaults/main.yml b/opt/infra/roles/nginx/defaults/main.yml
index cdd1d773..cdd1d773 100644
--- a/opt/servers/roles/nginx/defaults/main.yml
+++ b/opt/infra/roles/nginx/defaults/main.yml
diff --git a/opt/servers/roles/nginx/handlers/main.yml b/opt/infra/roles/nginx/handlers/main.yml
index 35585363..35585363 100644
--- a/opt/servers/roles/nginx/handlers/main.yml
+++ b/opt/infra/roles/nginx/handlers/main.yml
diff --git a/opt/servers/roles/nginx/tasks/main.yml b/opt/infra/roles/nginx/tasks/main.yml
index 5c78166c..5c78166c 100644
--- a/opt/servers/roles/nginx/tasks/main.yml
+++ b/opt/infra/roles/nginx/tasks/main.yml
diff --git a/opt/servers/roles/nginx/templates/elk.conf.j2 b/opt/infra/roles/nginx/templates/elk.conf.j2
index 271690e0..271690e0 100644
--- a/opt/servers/roles/nginx/templates/elk.conf.j2
+++ b/opt/infra/roles/nginx/templates/elk.conf.j2
diff --git a/opt/servers/roles/nginx/templates/qtip.conf.j2 b/opt/infra/roles/nginx/templates/qtip.conf.j2
index 3a15c6ba..3a15c6ba 100644
--- a/opt/servers/roles/nginx/templates/qtip.conf.j2
+++ b/opt/infra/roles/nginx/templates/qtip.conf.j2
diff --git a/opt/servers/roles/nginx/templates/testapi.conf.j2 b/opt/infra/roles/nginx/templates/testapi.conf.j2
index 6a4d388b..6a4d388b 100644
--- a/opt/servers/roles/nginx/templates/testapi.conf.j2
+++ b/opt/infra/roles/nginx/templates/testapi.conf.j2
diff --git a/opt/servers/roles/qtip/files/run_qtip_server.sh b/opt/infra/roles/qtip/files/run_qtip_server.sh
index 0f5cafea..0f5cafea 100644
--- a/opt/servers/roles/qtip/files/run_qtip_server.sh
+++ b/opt/infra/roles/qtip/files/run_qtip_server.sh
diff --git a/opt/servers/roles/qtip/tasks/main.yml b/opt/infra/roles/qtip/tasks/main.yml
index b8292791..b8292791 100644
--- a/opt/servers/roles/qtip/tasks/main.yml
+++ b/opt/infra/roles/qtip/tasks/main.yml
diff --git a/opt/servers/roles/testapi/files/run_testapi.sh b/opt/infra/roles/testapi/files/run_testapi.sh
index f9ba8387..f9ba8387 100644
--- a/opt/servers/roles/testapi/files/run_testapi.sh
+++ b/opt/infra/roles/testapi/files/run_testapi.sh
diff --git a/opt/servers/roles/testapi/tasks/main.yml b/opt/infra/roles/testapi/tasks/main.yml
index 8ca91396..8ca91396 100644
--- a/opt/servers/roles/testapi/tasks/main.yml
+++ b/opt/infra/roles/testapi/tasks/main.yml
diff --git a/opt/servers/roles/user/defaults/main.yml b/opt/infra/roles/user/defaults/main.yml
index ef496dd4..ef496dd4 100644
--- a/opt/servers/roles/user/defaults/main.yml
+++ b/opt/infra/roles/user/defaults/main.yml
diff --git a/opt/servers/roles/user/files/akhil.authorized_keys b/opt/infra/roles/user/files/akhil.authorized_keys
index 43942621..43942621 100644
--- a/opt/servers/roles/user/files/akhil.authorized_keys
+++ b/opt/infra/roles/user/files/akhil.authorized_keys
diff --git a/opt/servers/roles/user/files/serena.authorized_keys b/opt/infra/roles/user/files/serena.authorized_keys
index 5cdbfe00..5cdbfe00 100644
--- a/opt/servers/roles/user/files/serena.authorized_keys
+++ b/opt/infra/roles/user/files/serena.authorized_keys
diff --git a/opt/servers/roles/user/files/sudoers.d-qtip b/opt/infra/roles/user/files/sudoers.d-qtip
index e96d278a..e96d278a 100644
--- a/opt/servers/roles/user/files/sudoers.d-qtip
+++ b/opt/infra/roles/user/files/sudoers.d-qtip
diff --git a/opt/servers/roles/user/files/taseer.authorized_keys b/opt/infra/roles/user/files/taseer.authorized_keys
index 2efb95e8..2efb95e8 100644
--- a/opt/servers/roles/user/files/taseer.authorized_keys
+++ b/opt/infra/roles/user/files/taseer.authorized_keys
diff --git a/opt/servers/roles/user/files/yujunz.authorized_keys b/opt/infra/roles/user/files/yujunz.authorized_keys
index 83ed540d..83ed540d 100644
--- a/opt/servers/roles/user/files/yujunz.authorized_keys
+++ b/opt/infra/roles/user/files/yujunz.authorized_keys
diff --git a/opt/servers/roles/user/files/zhifeng.authorized_keys b/opt/infra/roles/user/files/zhifeng.authorized_keys
index 195cfdca..195cfdca 100644
--- a/opt/servers/roles/user/files/zhifeng.authorized_keys
+++ b/opt/infra/roles/user/files/zhifeng.authorized_keys
diff --git a/opt/servers/roles/user/tasks/main.yml b/opt/infra/roles/user/tasks/main.yml
index b1b5be9b..b1b5be9b 100644
--- a/opt/servers/roles/user/tasks/main.yml
+++ b/opt/infra/roles/user/tasks/main.yml
diff --git a/opt/servers/test.yml b/opt/infra/test.yml
index ee3ded13..ee3ded13 100644
--- a/opt/servers/test.yml
+++ b/opt/infra/test.yml
diff --git a/opt/plugin/__init__.py b/opt/plugin/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/opt/plugin/__init__.py
diff --git a/opt/plugin/doctor/__init__.py b/opt/plugin/doctor/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/opt/plugin/doctor/__init__.py
diff --git a/opt/plugin/doctor/collector.yaml b/opt/plugin/doctor/collector.yaml
new file mode 100644
index 00000000..40167588
--- /dev/null
+++ b/opt/plugin/doctor/collector.yaml
@@ -0,0 +1,24 @@
+# collector configuration for doctor verification job
+# collect performance data from log files
+collector: logfile
+# transform collected data into timeline
+transformer: timeline
+logfiles:
+  - filename: doctor_consumer.log
+  # 2016-12-28 03:16:05,630 consumer.py 26 INFO   doctor consumer notified at 1482894965.63
+    - match: doctor consumer notified at \d+(\.\d+)?$
+      checkpoint: notified consumer
+  - filename: doctor_inspector.log
+  # 2016-12-28 03:16:05,299 inspector.py 76 INFO   event posted at 1482894965.3
+  # 2016-12-28 03:16:05,299 inspector.py 56 INFO   doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
+  # 2016-12-28 03:16:05,506 inspector.py 66 INFO   doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
+    - match: event posted at \d+(\.\d+)?$
+      checkpoint: posted event
+    - match: doctor mark vm\(.*\) error at \d+(\.\d+)?$
+      checkpoint: marked VM error
+    - match: doctor mark host\(.*\) down at \d+(\.\d+)?$
+      checkpoint: marked host down
+  - filename: disable_network.log
+  # doctor set host down at 1482894965.164096803
+    - match: doctor set host down at \d+(\.\d+)?$
+    - checkpoint: set host down
diff --git a/opt/plugin/doctor/sample/__init__.py b/opt/plugin/doctor/sample/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/opt/plugin/doctor/sample/__init__.py
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log
new file mode 100755
index 00000000..8c577b59
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log
@@ -0,0 +1,4 @@
+ * Running on http://0.0.0.0:12346/ (Press CTRL+C to quit)
+2016-12-28 03:16:05,630 consumer.py 26 INFO   doctor consumer notified at 1482894965.63
+2016-12-28 03:16:05,630 consumer.py 27 INFO   received data = {"severity": "moderate", "alarm_name": "doctor_alarm1", "current": "alarm", "alarm_id": "d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97", "reason": "Event <id=e27a3560-7abd-41e2-9591-b2c15baa3e94,event_type=compute.instance.update> hits the query <query=[{\"field\": \"traits.state\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"error\"}, {\"field\": \"traits.instance_id\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"51c0739a-0b87-4dd7-8d22-ff7fd8af04bb\"}]>.", "reason_data": {"type": "event", "event": {"event_type": "compute.instance.update", "traits": [["state", 1, "error"], ["user_id", 1, "c844f5397a924d908f1246515e4843e6"], ["service", 1, "compute"], ["disk_gb", 2, 1], ["instance_type", 1, "m1.tiny"], ["tenant_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["root_gb", 2, 1], ["ephemeral_gb", 2, 0], ["instance_type_id", 2, 3], ["vcpus", 2, 1], ["memory_mb", 2, 512], ["instance_id", 1, "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb"], ["host", 1, "overcloud-controller-2.ool-virtual1"], ["request_id", 1, "req-531a60d6-ee47-4fb1-8d86-9e42425d6427"], ["project_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["launched_at", 4, "2016-12-28T03:14:43"]], "message_signature": "b784003e35c0ec531c44dc33836b4b4a8d4544351894c12951ef946edf18bb8f", "raw": {}, "generated": "2016-12-28T03:16:05.494044", "message_id": "e27a3560-7abd-41e2-9591-b2c15baa3e94"}}, "previous": "insufficient data"}
+127.0.0.1 - - [28/Dec/2016 03:16:05] "POST /failure HTTP/1.1" 200 -
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log
new file mode 100755
index 00000000..1b488491
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log
@@ -0,0 +1,11 @@
+++ sudo ip a
+++ awk '/ 192.0.2.5\//{print $7}'
++ dev=eth0
++ sleep 1
+++ date +%s.%N
++ echo 'doctor set host down at' 1482894965.164096803
+doctor set host down at 1482894965.164096803
++ sudo ip link set eth0 down
++ sleep 180
++ sudo ip link set eth0 up
++ sleep 1
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log
new file mode 100755
index 00000000..e040e809
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log
@@ -0,0 +1,2 @@
+2016-12-28 03:16:05,630 consumer.py 26 INFO   doctor consumer notified at 1482894965.63
+2016-12-28 03:16:05,630 consumer.py 27 INFO   received data = {"severity": "moderate", "alarm_name": "doctor_alarm1", "current": "alarm", "alarm_id": "d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97", "reason": "Event <id=e27a3560-7abd-41e2-9591-b2c15baa3e94,event_type=compute.instance.update> hits the query <query=[{\"field\": \"traits.state\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"error\"}, {\"field\": \"traits.instance_id\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"51c0739a-0b87-4dd7-8d22-ff7fd8af04bb\"}]>.", "reason_data": {"type": "event", "event": {"event_type": "compute.instance.update", "traits": [["state", 1, "error"], ["user_id", 1, "c844f5397a924d908f1246515e4843e6"], ["service", 1, "compute"], ["disk_gb", 2, 1], ["instance_type", 1, "m1.tiny"], ["tenant_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["root_gb", 2, 1], ["ephemeral_gb", 2, 0], ["instance_type_id", 2, 3], ["vcpus", 2, 1], ["memory_mb", 2, 512], ["instance_id", 1, "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb"], ["host", 1, "overcloud-controller-2.ool-virtual1"], ["request_id", 1, "req-531a60d6-ee47-4fb1-8d86-9e42425d6427"], ["project_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["launched_at", 4, "2016-12-28T03:14:43"]], "message_signature": "b784003e35c0ec531c44dc33836b4b4a8d4544351894c12951ef946edf18bb8f", "raw": {}, "generated": "2016-12-28T03:16:05.494044", "message_id": "e27a3560-7abd-41e2-9591-b2c15baa3e94"}}, "previous": "insufficient data"}
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log
new file mode 100755
index 00000000..e2fd9477
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log
@@ -0,0 +1,6 @@
+2016-12-28 03:15:01,868 inspector.py 50 DEBUG  get hostname=overcloud-novacompute-1.ool-virtual1 from server=<Server: doctor_vm1>
+2016-12-28 03:16:05,299 inspector.py 76 INFO   event posted at 1482894965.3
+2016-12-28 03:16:05,299 inspector.py 77 INFO   inspector = <__main__.DoctorInspectorSample object at 0x7fcce167aa90>
+2016-12-28 03:16:05,299 inspector.py 78 INFO   received data = {"hostname": "overcloud-novacompute-1.ool-virtual1", "type": "compute.host.down"}
+2016-12-28 03:16:05,299 inspector.py 56 INFO   doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
+2016-12-28 03:16:05,506 inspector.py 66 INFO   doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log
new file mode 100755
index 00000000..a4c6ec8f
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log
@@ -0,0 +1,3 @@
+2016-12-28 03:15:00,890 monitor.py 65 DEBUG  start ping to host overcloud-novacompute-1.ool-virtual1 (ip=192.0.2.5)
+2016-12-28 03:16:05,292 monitor.py 74 INFO   doctor monitor detected at 1482894965.29
+2016-12-28 03:16:05,960 monitor.py 76 INFO   ping timeout, quit monitoring...
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log
new file mode 100755
index 00000000..2933ad61
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log
@@ -0,0 +1,421 @@
++ IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img
++ IMAGE_NAME=cirros
++ IMAGE_FILE=cirros.img
++ IMAGE_FORMAT=qcow2
++ VM_NAME=doctor_vm1
++ VM_FLAVOR=m1.tiny
++ ALARM_NAME=doctor_alarm1
++ INSPECTOR_PORT=12345
++ CONSUMER_PORT=12346
++ DOCTOR_USER=doctor
++ DOCTOR_PW=doctor
++ DOCTOR_PROJECT=doctor
++ DOCTOR_ROLE=admin
++++ dirname ./run.sh
+++ cd .
+++ pwd
++ TOP_DIR=/home/opnfv/repos/doctor/tests
++ ssh_opts='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
++ as_doctor_user='--os-username doctor --os-password doctor
+                --os-tenant-name doctor'
++ echo 'Note: doctor/tests/run.sh has been executed.'
+Note: doctor/tests/run.sh has been executed.
++ trap cleanup EXIT
++ source /home/opnfv/repos/doctor/tests/functions-common
++ source /home/opnfv/repos/doctor/tests/lib/installer
+++ INSTALLER_TYPE=apex
+++ INSTALLER_IP=192.168.122.70
+++ ssh_opts_cpu='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
++ source /home/opnfv/repos/doctor/tests/lib/inspector
+++ INSPECTOR_TYPE=sample
++ setup_installer
++ is_installer_supported apex
++ local installer=apex
++ [[ -f /home/opnfv/repos/doctor/tests/lib/installers/apex ]]
++ source /home/opnfv/repos/doctor/tests/lib/installers/apex
++ is_set INSTALLER_IP
++ local 'var=$INSTALLER_IP'
++ eval '[ -n "$INSTALLER_IP" ]'
+++ '[' -n 192.168.122.70 ']'
++ installer_get_ssh_keys
++ sudo scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@192.168.122.70:/home/stack/.ssh/id_rsa instack_key
+Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.

+++ whoami
+++ whoami
++ sudo chown root:root instack_key
++ chmod 400 instack_key
++ ssh_opts_cpu+=' -i instack_key'
++ installer_apply_patches
++ return
++ echo 'preparing VM image...'
+preparing VM image...
++ download_image
++ use_existing_image=false
++ openstack image list
++ grep -q ' cirros '
++ use_existing_image=true
++ [[ true == false ]]
++ register_image
++ openstack image list
++ grep -q ' cirros '
++ return 0
++ echo 'creating test user...'
+creating test user...
++ create_test_user
++ openstack project list
++ grep -q ' doctor '
++ openstack project create doctor
++-------------+----------------------------------+
+| Field       | Value                            |
++-------------+----------------------------------+
+| description | None                             |
+| enabled     | True                             |
+| id          | dffffadad42d41d5a618570f52e3305f |
+| name        | doctor                           |
++-------------+----------------------------------+
++ openstack user list
++ grep -q ' doctor '
++ openstack user create doctor --password doctor --project doctor
++------------+----------------------------------+
+| Field      | Value                            |
++------------+----------------------------------+
+| email      | None                             |
+| enabled    | True                             |
+| id         | c844f5397a924d908f1246515e4843e6 |
+| name       | doctor                           |
+| project_id | dffffadad42d41d5a618570f52e3305f |
+| username   | doctor                           |
++------------+----------------------------------+
++ openstack user role list doctor --project doctor
++ grep -q ' admin '
++ openstack role add admin --user doctor --project doctor
++-----------+----------------------------------+
+| Field     | Value                            |
++-----------+----------------------------------+
+| domain_id | None                             |
+| id        | 4b0d10e07705433c865958471d7c4ff5 |
+| name      | admin                            |
++-----------+----------------------------------+
++ echo 'creating VM...'
+creating VM...
++ boot_vm
++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
++ grep -q ' doctor_vm1 '
++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server create --flavor m1.tiny --image cirros doctor_vm1
++--------------------------------------+------------------------------------------------+
+| Field                                | Value                                          |
++--------------------------------------+------------------------------------------------+
+| OS-DCF:diskConfig                    | MANUAL                                         |
+| OS-EXT-AZ:availability_zone          |                                                |
+| OS-EXT-SRV-ATTR:host                 | None                                           |
+| OS-EXT-SRV-ATTR:hypervisor_hostname  | None                                           |
+| OS-EXT-SRV-ATTR:instance_name        |                                                |
+| OS-EXT-STS:power_state               | 0                                              |
+| OS-EXT-STS:task_state                | scheduling                                     |
+| OS-EXT-STS:vm_state                  | building                                       |
+| OS-SRV-USG:launched_at               | None                                           |
+| OS-SRV-USG:terminated_at             | None                                           |
+| accessIPv4                           |                                                |
+| accessIPv6                           |                                                |
+| addresses                            |                                                |
+| adminPass                            | jL7t3S3DUrCX                                   |
+| config_drive                         |                                                |
+| created                              | 2016-12-28T03:14:37Z                           |
+| flavor                               | m1.tiny (7f924f9f-2fa0-40d4-95fb-53079307ec66) |
+| hostId                               |                                                |
+| id                                   | 51c0739a-0b87-4dd7-8d22-ff7fd8af04bb           |
+| image                                | cirros (3071d9bf-7be5-4cb1-a706-428daa446ec3)  |
+| key_name                             | None                                           |
+| name                                 | doctor_vm1                                     |
+| os-extended-volumes:volumes_attached | []                                             |
+| progress                             | 0                                              |
+| project_id                           | dffffadad42d41d5a618570f52e3305f               |
+| properties                           |                                                |
+| security_groups                      | [{u'name': u'default'}]                        |
+| status                               | BUILD                                          |
+| updated                              | 2016-12-28T03:14:37Z                           |
+| user_id                              | c844f5397a924d908f1246515e4843e6               |
++--------------------------------------+------------------------------------------------+
++ sleep 1
++ wait_for_vm_launch
++ echo 'waiting for vm launch...'
+waiting for vm launch...
++ count=0
++ [[ 0 -lt 60 ]]
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
+++ grep ' doctor_vm1 '
+++ awk '{print $6}'
++ state=BUILD
++ [[ BUILD == \A\C\T\I\V\E ]]
++ [[ BUILD == \E\R\R\O\R ]]
++ count=1
++ sleep 1
++ [[ 1 -lt 60 ]]
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
+++ grep ' doctor_vm1 '
+++ awk '{print $6}'
++ state=ACTIVE
++ [[ ACTIVE == \A\C\T\I\V\E ]]
++ sleep 5
++ return 0
++ echo 'get computer host info...'
+get computer host info...
++ get_compute_host_info
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server show doctor_vm1
+++ grep OS-EXT-SRV-ATTR:host
+++ awk '{ print $4 }'
++ COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
++ compute_host_in_undercloud=overcloud-novacompute-1
++ die_if_not_set 45 COMPUTE_HOST 'Failed to get compute hostname'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ is_installer apex
++ local installer=apex
++ [[ apex == apex ]]
++ COMPUTE_USER=heat-admin
+++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192.168.122.70 'source stackrc;              nova show overcloud-novacompute-1              | awk '\''/ ctlplane network /{print $5}'\'''
+Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.

++ COMPUTE_IP=192.0.2.5
++ die_if_not_set 63 COMPUTE_IP 'Could not resolve overcloud-novacompute-1.ool-virtual1. Either manually set COMPUTE_IP or enable DNS resolution.'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ echo COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
+COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
++ echo COMPUTE_IP=192.0.2.5
+COMPUTE_IP=192.0.2.5
++ ping -c 1 192.0.2.5
+PING 192.0.2.5 (192.0.2.5) 56(84) bytes of data.
+64 bytes from 192.0.2.5: icmp_seq=1 ttl=63 time=0.275 ms
+
+--- 192.0.2.5 ping statistics ---
+1 packets transmitted, 1 received, 0% packet loss, time 0ms
+rtt min/avg/max/mdev = 0.275/0.275/0.275/0.000 ms
++ [[ 0 -ne 0 ]]
++ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5 exit
+Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.

++ [[ 0 -ne 0 ]]
++ echo 'creating alarm...'
+creating alarm...
++ create_alarm
++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-list
++ grep -q ' doctor_alarm1 '
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
+++ grep ' doctor_vm1 '
+++ awk '{print $2}'
++ vm_id=51c0739a-0b87-4dd7-8d22-ff7fd8af04bb
++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-event-create --name doctor_alarm1 --alarm-action http://localhost:12346/failure --description 'VM failure' --enabled True --repeat-actions False --severity moderate --event-type compute.instance.update -q 'traits.state=string::error; traits.instance_id=string::51c0739a-0b87-4dd7-8d22-ff7fd8af04bb'
++---------------------------+----------------------------------------------------------------------+
+| Property                  | Value                                                                |
++---------------------------+----------------------------------------------------------------------+
+| alarm_actions             | ["http://localhost:12346/failure"]                                   |
+| alarm_id                  | d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97                                 |
+| description               | VM failure                                                           |
+| enabled                   | True                                                                 |
+| event_type                | compute.instance.update                                              |
+| insufficient_data_actions | []                                                                   |
+| name                      | doctor_alarm1                                                        |
+| ok_actions                | []                                                                   |
+| project_id                | dffffadad42d41d5a618570f52e3305f                                     |
+| query                     | [{"field": "traits.state", "type": "string", "value": "error", "op": |
+|                           | "eq"}, {"field": "traits.instance_id", "type": "string", "value":    |
+|                           | "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb", "op": "eq"}]                 |
+| repeat_actions            | False                                                                |
+| severity                  | moderate                                                             |
+| state                     | insufficient data                                                    |
+| type                      | event                                                                |
+| user_id                   | c844f5397a924d908f1246515e4843e6                                     |
++---------------------------+----------------------------------------------------------------------+
++ echo 'starting doctor sample components...'
+starting doctor sample components...
++ start_inspector
++ is_inspector_supported sample
++ local inspector=sample
++ [[ -f /home/opnfv/repos/doctor/tests/lib/inspectors/sample ]]
++ source /home/opnfv/repos/doctor/tests/lib/inspectors/sample
++ start_inspector_sample
++ pgrep -f 'python inspector.py'
++ start_monitor
++ pgrep -f 'python monitor.py'
++ python inspector.py 12345
++ start_consumer
++ sudo -E python monitor.py overcloud-novacompute-1.ool-virtual1 192.0.2.5 sample http://127.0.0.1:12345/events
++ pgrep -f 'python consumer.py'
++ is_installer local
++ local installer=local
++ python consumer.py 12346
++ [[ local == apex ]]
++ is_installer apex
++ local installer=apex
++ [[ apex == apex ]]
+++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192.168.122.70 'source stackrc;                              nova list | grep '\'' overcloud-controller-[0-9] '\''                              | sed -e '\''s/^.*ctlplane=//'\'' -e '\''s/ *|$//'\'''
+Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.

++ CONTROLLER_IPS='192.0.2.6
+192.0.2.8
+192.0.2.9'
++ die_if_not_set 185 CONTROLLER_IPS 'Could not get CONTROLLER_IPS.'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600
++ sleep 60
++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600
++ echo 'injecting host failure...'
+injecting host failure...
++ inject_failure
++ echo 'disabling network of compute host [overcloud-novacompute-1.ool-virtual1] for 3 mins...'
+disabling network of compute host [overcloud-novacompute-1.ool-virtual1] for 3 mins...
++ cat
++ sed -i -e s/@COMPUTE_IP@/192.0.2.5/ disable_network.sh
++ chmod +x disable_network.sh
++ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key disable_network.sh heat-admin@192.0.2.5:
+Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.

++ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
+Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.

++ sleep 60
++ check_host_status '(DOWN|UNKNOWN)'
++ expected_state='(DOWN|UNKNOWN)'
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor --os-compute-api-version 2.16 server show doctor_vm1
+++ grep host_status
++ host_status_line='| host_status                          | DOWN                                                     |'
+++ echo '|' host_status '|' DOWN '|'
+++ awk '{print $4}'
++ host_status=DOWN
++ die_if_not_set 283 host_status 'host_status not reported by: nova show doctor_vm1'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ [[ (DOWN|UNKNOWN) =~ DOWN ]]
++ echo 'doctor_vm1 showing host_status: DOWN'
+doctor_vm1 showing host_status: DOWN
++ calculate_notification_time
+++ grep 'doctor monitor detected at' monitor.log
+++ awk '{print $10}'
++ detected=1482894965.29
+++ grep 'doctor consumer notified at' consumer.log
+++ awk '{print $10}'
++ notified=1482894965.63
++ grep -q 'doctor consumer notified at' consumer.log
++ [[ PROFILER == \p\o\c ]]
++ echo '1482894965.63 1482894965.29'
++ awk '{
+            d = $1 - $2;
+            if (d < 1 && d > 0) { print d " OK"; exit 0 }
+            else { print d " NG"; exit 1 }
+        }'
+0.34 OK
++ echo done
+done
++ cleanup
++ set +e
++ echo cleanup...
+cleanup...
++ stop_monitor
++ pgrep -f 'python monitor.py'
++ return 0
++ stop_inspector
++ stop_inspector_sample
++ pgrep -f 'python inspector.py'
+377
+++ pgrep -f 'python inspector.py'
++ kill 377
++ stop_consumer
++ pgrep -f 'python consumer.py'
+381
+++ pgrep -f 'python consumer.py'
++ kill 381
++ is_installer local
++ local installer=local
++ [[ local == apex ]]
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
+++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
++ kill 398
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
+./run.sh: line 206:   377 Terminated              python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1
+./run.sh: line 206:   381 Terminated              python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1
+++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
++ kill 399
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
+++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
++ kill 400
++ echo 'waiting disabled compute host back to be enabled...'
+waiting disabled compute host back to be enabled...
++ python ./nova_force_down.py overcloud-novacompute-1.ool-virtual1 --unset
+<Response [200]>
++ sleep 240
++ check_host_status UP
++ expected_state=UP
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor --os-compute-api-version 2.16 server show doctor_vm1
+++ grep host_status
++ host_status_line='| host_status                          | UP                                                       |'
+++ echo '|' host_status '|' UP '|'
+++ awk '{print $4}'
++ host_status=UP
++ die_if_not_set 283 host_status 'host_status not reported by: nova show doctor_vm1'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ [[ UP =~ UP ]]
++ echo 'doctor_vm1 showing host_status: UP'
+doctor_vm1 showing host_status: UP
++ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5:disable_network.log .
+Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.

++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
++ grep -q ' doctor_vm1 '
++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server delete doctor_vm1
++ sleep 1
+++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-list
+++ grep ' doctor_alarm1 '
+++ awk '{print $2}'
++ alarm_id=d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97
++ sleep 1
++ '[' -n d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 ']'
++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-delete d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97
++ sleep 1
+++ openstack image list
+++ grep ' cirros '
+++ awk '{print $2}'
++ image_id=3071d9bf-7be5-4cb1-a706-428daa446ec3
++ sleep 1
++ [[ true == false ]]
++ openstack role remove admin --user doctor --project doctor
++ openstack project delete doctor
++ openstack user delete doctor
++ cleanup_installer
++ cleanup_installer_apex
++ return
++ cleanup_inspector
++ cleanup_inspector_sample
++ return
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log
new file mode 100755
index 00000000..0d07de02
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log
@@ -0,0 +1,9 @@
+2016-12-28 03:15:01,868 inspector.py 50 DEBUG  get hostname=overcloud-novacompute-1.ool-virtual1 from server=<Server: doctor_vm1>
+ * Running on http://127.0.0.1:12345/ (Press CTRL+C to quit)
+2016-12-28 03:16:05,299 inspector.py 76 INFO   event posted at 1482894965.3
+2016-12-28 03:16:05,299 inspector.py 77 INFO   inspector = <__main__.DoctorInspectorSample object at 0x7fcce167aa90>
+2016-12-28 03:16:05,299 inspector.py 78 INFO   received data = {"hostname": "overcloud-novacompute-1.ool-virtual1", "type": "compute.host.down"}
+2016-12-28 03:16:05,299 inspector.py 56 INFO   doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
+2016-12-28 03:16:05,506 inspector.py 66 INFO   doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
+<Response [200]>
+127.0.0.1 - - [28/Dec/2016 03:16:05] "POST /events HTTP/1.1" 200 -
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log
new file mode 100755
index 00000000..a4c6ec8f
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log
@@ -0,0 +1,3 @@
+2016-12-28 03:15:00,890 monitor.py 65 DEBUG  start ping to host overcloud-novacompute-1.ool-virtual1 (ip=192.0.2.5)
+2016-12-28 03:16:05,292 monitor.py 74 INFO   doctor monitor detected at 1482894965.29
+2016-12-28 03:16:05,960 monitor.py 76 INFO   ping timeout, quit monitoring...
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log
new file mode 100755
index 00000000..70cc8945
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log
@@ -0,0 +1 @@
+Warning: Permanently added '192.0.2.6' (ECDSA) to the list of known hosts.

diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log
new file mode 100755
index 00000000..54222d2a
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log
@@ -0,0 +1 @@
+Warning: Permanently added '192.0.2.8' (ECDSA) to the list of known hosts.

diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log
new file mode 100755
index 00000000..2660035b
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log
@@ -0,0 +1 @@
+Warning: Permanently added '192.0.2.9' (ECDSA) to the list of known hosts.

diff --git a/qtip/api/cmd/server.py b/qtip/api/cmd/server.py
index 3b89af1e..eea45ad3 100644
--- a/qtip/api/cmd/server.py
+++ b/qtip/api/cmd/server.py
@@ -10,7 +10,8 @@
 from flask import Flask
 from flask_restful import Api
 from flask_restful_swagger import swagger
-import qtip.api.router.mapper as mapper
+
+import legacy.api.router.mapper as mapper
 
 app = Flask(__name__)
 api = swagger.docs(Api(app), apiVersion='0.1', description='QTIP API specs')
diff --git a/qtip/base/benchmark.py b/qtip/base/benchmark.py
deleted file mode 100644
index b38e6016..00000000
--- a/qtip/base/benchmark.py
+++ /dev/null
@@ -1,95 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from itertools import chain
-from os import listdir
-from os import path
-import yaml
-
-
-class Property(object):
-    # list
-    NAME = 'name'
-    CONTENT = 'content'
-    ABSPATH = 'abspath'
-    # content
-    TITLE = 'title'
-    DESCRIPTION = 'description'
-    # spec
-    ALGORITHM = 'algorithm'
-    SECTIONS = 'sections'
-    WEIGHT = 'weight'
-    METRICS = 'metrics'
-    SPEC = 'spec'
-    WORKLOADS = 'workloads'
-    # plan
-    INFO = 'info'
-    FACILITY = 'facility'
-    ENGINEER = 'engineer'
-    SUITES = 'suites'
-    # suite
-    QPI_SPEC = 'QPI_spec'
-    CONDITION = 'condition'
-    CASES = 'cases'
-    # case
-    METRIC_SPEC = 'metric_spec'
-    CONFIG = 'config'
-
-
-class QtipError(Exception):
-    pass
-
-
-class Algorithm(object):
-    ARITHMETIC_MEAN = 'arithmetic mean'
-    WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
-    GEOMETRIC_MEAN = 'geometric mean'
-    WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
-
-
-ROOT_DIR = 'benchmarks'
-
-
-class Benchmark(object):
-    """Abstract class of QTIP benchmarks"""
-    DEFAULT_DIR = '.'
-    _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
-                        ROOT_DIR)]
-
-    def __init__(self, name, paths=None):
-        self._file = name
-        self._abspath = self._find(name, paths)
-        self.name = path.splitext(name)[0]
-
-    def _find(self, name, paths):
-        """find a benchmark in searching paths"""
-        paths = self._paths if paths is None else paths
-        name = path.join(self.DEFAULT_DIR, name)
-        for p in paths:
-            abspath = path.join(p, name)
-            if path.exists(abspath):
-                return abspath
-        raise QtipError("'{}' not found in paths: {}".format(name, paths))
-
-    @classmethod
-    def list_all(cls, paths=None):
-        """list all available benchmarks"""
-        paths = cls._paths if paths is None else paths
-        names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR))
-                                     for p in paths])
-        for name in names:
-            item = cls(name, paths=paths)
-            yield {
-                Property.NAME: name,
-                Property.ABSPATH: item._abspath,
-                Property.CONTENT: item.content()}
-
-    def content(self):
-        """description of benchmark"""
-        return yaml.safe_load(file(self._abspath))
diff --git a/qtip/base/constant.py b/qtip/base/constant.py
new file mode 100644
index 00000000..76481b47
--- /dev/null
+++ b/qtip/base/constant.py
@@ -0,0 +1,50 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class FormulaName(object):
+    """formula names"""
+    ARITHMETIC_MEAN = 'arithmetic mean'
+    WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
+    GEOMETRIC_MEAN = 'geometric mean'
+    WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
+
+
+class PkgName(object):
+    """QTIP package names"""
+    COLLECTOR = 'collector'
+    DRIVER = 'driver'
+    REPORTER = 'reporter'
+    RUNNER = 'runner'
+    SPEC = 'spec'
+
+
+class PropName(object):
+    """property names"""
+    # list
+    NAME = 'name'
+    CONTENT = 'content'
+    ABSPATH = 'abspath'
+    # content
+    name = 'name'
+    DESCRIPTION = 'description'
+    # spec
+    SECTIONS = 'sections'
+    WEIGHT = 'weight'
+    FORMULA = 'formula'
+    METRICS = 'metrics'
+    WORKLOADS = 'workloads'
+    # plan
+    CONFIG = 'config'
+    FACILITY = 'facility'
+    ENGINEER = 'engineer'
+    DRIVER = 'driver'
+    COLLECTOR = 'collector'
+    REPORTER = 'reporter'
+    QPIS = 'QPIs'
diff --git a/qtip/base/error.py b/qtip/base/error.py
new file mode 100644
index 00000000..01a7f7a6
--- /dev/null
+++ b/qtip/base/error.py
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class QtipError(Exception):
+    pass
+
+
+class InvalidFormat(QtipError):
+    def __init__(self, filename):
+        self.filename = filename
+
+
+class NotFound(QtipError):
+    def __init__(self, module, package='qtip'):
+        self.package = package
+        self.module = module
+
+
+class ToBeDoneError(QtipError):
+    """something still to be done"""
+    def __init__(self, method, module):
+        self.method = method
+        self.module = module
+
+
+def make_tbd(method, module='qtip'):
+    def tbd():
+        raise ToBeDoneError(method, module)
+    return tbd
diff --git a/qtip/runner/case.py b/qtip/cli/commands/cmd_metric.py
index eb3febc2..7f5c18d3 100644
--- a/qtip/runner/case.py
+++ b/qtip/cli/commands/cmd_metric.py
@@ -7,11 +7,27 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
+import click
 
+from qtip.cli.entry import Context
 
-class Case(object):
-    def __init__(self, spec, paths=None):
-        self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths)
-        self.config = spec[Property.CONFIG]
+pass_context = click.make_pass_decorator(Context, ensure=False)
+
+
+@click.group()
+@pass_context
+def cli(ctx):
+    ''' Performance Metrics Group '''
+    pass
+
+
+@cli.command('list', help='List all the Metric Groups')
+@pass_context
+def cmd_list(ctx):
+    pass
+
+
+@cli.command('run', help='Run tests to collect Performance Metrics')
+@pass_context
+def cmd_run(ctx):
+    pass
diff --git a/qtip/cli/commands/cmd_perftest.py b/qtip/cli/commands/cmd_perftest.py
deleted file mode 100644
index 129ee2d9..00000000
--- a/qtip/cli/commands/cmd_perftest.py
+++ /dev/null
@@ -1,58 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from prettytable import PrettyTable
-import yaml
-import click
-import os
-from qtip.cli import helper
-
-
-class PerfTest:
-
-    def __init__(self):
-        self.path = os.path.join(helper.fetch_root(), 'perftest/summary')
-
-    def list(self):
-        table = PrettyTable(["Name", "Description"])
-        table.align = 'l'
-        with open(self.path) as tests:
-            line = tests.read()
-            data = yaml.safe_load(line)['test_cases']
-            for i in range(0, len(data)):
-                points = data[i]
-                table.add_row([points['name'], points['description']])
-        click.echo(table)
-
-    def run(self):
-        click.echo("Run a perftest")
-
-
-@click.group()
-def cli():
-    pass
-
-
-@cli.group()
-@click.pass_context
-def perftest(ctx):
-    pass
-
-
-_perftest = PerfTest()
-
-
-@perftest.command("list", help="Lists all perftest benchmarks.")
-def list():
-    _perftest.list()
-
-
-@perftest.command("run", help="Executes a single perftest benchmark.")
-def execute():
-    _perftest.run()
diff --git a/qtip/cli/commands/cmd_plan.py b/qtip/cli/commands/cmd_plan.py
index 01bf8251..6f622e5a 100644
--- a/qtip/cli/commands/cmd_plan.py
+++ b/qtip/cli/commands/cmd_plan.py
@@ -9,7 +9,8 @@
 
 import click
 from prettytable import PrettyTable
-from qtip.runner.plan import Plan
+
+from qtip.loader.plan import Plan
 
 
 @click.group()
diff --git a/qtip/collector/__init__.py b/qtip/collector/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/collector/__init__.py
diff --git a/qtip/drivers/base.py b/qtip/collector/base.py
index 1aa8d8ad..cd8fc797 100644
--- a/qtip/drivers/base.py
+++ b/qtip/collector/base.py
@@ -8,5 +8,5 @@
 ##############################################################################
 
 
-class BaseDriver(object):
-    """performance testing tool driver"""
+class BaseCollector(object):
+    """performance metrics collector"""
diff --git a/qtip/spec/qpi.py b/qtip/collector/logfile.py
index b7d7aa02..6528ea9f 100644
--- a/qtip/spec/qpi.py
+++ b/qtip/collector/logfile.py
@@ -7,12 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.base.benchmark import Benchmark
+from base import BaseCollector
 
 
-class QPISpec(Benchmark):
-    """
-    a QPI specification defines how to calculate a performance index from
-     collected metrics.
-    """
-    DEFAULT_DIR = 'QPI'
+class LogfileCollector(BaseCollector):
+    """collect performance metrics from log files"""
diff --git a/qtip/collector/transformer/__init__.py b/qtip/collector/transformer/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/collector/transformer/__init__.py
diff --git a/qtip/agent/collector.py b/qtip/collector/transformer/base.py
index 3df21379..bf38c493 100644
--- a/qtip/agent/collector.py
+++ b/qtip/collector/transformer/base.py
@@ -8,6 +8,5 @@
 ##############################################################################
 
 
-class Collector(object):
-    """collect test result and test condition"""
-    pass
+class BaseTransformer(object):
+    """Transform collected performance metrics into required format"""
diff --git a/qtip/collector/transformer/timeline.py b/qtip/collector/transformer/timeline.py
new file mode 100644
index 00000000..9dd5850c
--- /dev/null
+++ b/qtip/collector/transformer/timeline.py
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from base import BaseTransformer
+
+
+class Timeline(BaseTransformer):
+    """transform collected metrics into object required by timeline template"""
diff --git a/qtip/driver/__init__.py b/qtip/driver/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/driver/__init__.py
diff --git a/qtip/drivers/ansible.py b/qtip/driver/ansible.py
index 04e9f9bd..04e9f9bd 100644
--- a/qtip/drivers/ansible.py
+++ b/qtip/driver/ansible.py
diff --git a/qtip/driver/base.py b/qtip/driver/base.py
new file mode 100644
index 00000000..6f5cab3c
--- /dev/null
+++ b/qtip/driver/base.py
@@ -0,0 +1,20 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseDriver(object):
+    """performance testing tool driver"""
+    def pre_run(self):
+        pass
+
+    def run(self):
+        pass
+
+    def post_run(self):
+        pass
diff --git a/qtip/driver/sample.py b/qtip/driver/sample.py
new file mode 100644
index 00000000..9b347949
--- /dev/null
+++ b/qtip/driver/sample.py
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from base import BaseDriver
+
+
+class SampleDriver(BaseDriver):
+    """sample driver that generates random data for testing"""
diff --git a/qtip/drivers/yardstick.py b/qtip/driver/yardstick.py
index 83f1b3d8..83f1b3d8 100644
--- a/qtip/drivers/yardstick.py
+++ b/qtip/driver/yardstick.py
diff --git a/qtip/loader/__init__.py b/qtip/loader/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/loader/__init__.py
diff --git a/qtip/loader/base.py b/qtip/loader/base.py
new file mode 100644
index 00000000..d3a7e1d3
--- /dev/null
+++ b/qtip/loader/base.py
@@ -0,0 +1,62 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from itertools import chain
+from os import listdir
+from os import path
+import yaml
+
+from qtip.base.error import InvalidFormat, NotFound
+from qtip.base.constant import PropName
+
+
+ROOT_DIR = path.join(path.dirname(__file__), path.pardir, path.pardir,
+                     'benchmarks')
+
+
+class BaseLoader(object):
+    """Abstract class of QTIP benchmark loader"""
+    RELATIVE_PATH = '.'
+    _paths = [ROOT_DIR]
+
+    def __init__(self, name, paths=None):
+        self._file = name
+        self._abspath = self._find(name, paths=paths)
+
+        try:
+            content = yaml.safe_load(file(self._abspath))
+        except yaml.YAMLError:
+            # TODO(yujunz) log yaml error
+            raise InvalidFormat(self._abspath)
+
+        self.name = content[PropName.NAME] if PropName.NAME in content \
+            else path.splitext(name)[0]
+        self.content = content
+
+    def _find(self, name, paths=None):
+        """find a benchmark in searching paths"""
+        paths = self._paths if paths is None else paths
+        for p in paths:
+            abspath = path.join(p, self.RELATIVE_PATH, name)
+            if path.exists(abspath):
+                return abspath
+        raise NotFound(name, paths)
+
+    @classmethod
+    def list_all(cls, paths=None):
+        """list all available benchmarks"""
+        paths = cls._paths if paths is None else paths
+        names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH))
+                                     for p in paths])
+        for name in names:
+            item = cls(name, paths=paths)
+            yield {
+                PropName.NAME: name,
+                PropName.ABSPATH: item._abspath,
+                PropName.CONTENT: item.content}
diff --git a/qtip/spec/metric.py b/qtip/loader/metric.py
index e9c70547..8b6fa5d3 100644
--- a/qtip/spec/metric.py
+++ b/qtip/loader/metric.py
@@ -7,10 +7,10 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.base.benchmark import Benchmark
+from base import BaseLoader
 
 
-class MetricSpec(Benchmark):
+class MetricSpec(BaseLoader):
     """metrics in QTIP are categorized by performance test tools, such as
     dhrystone, whetstone and etc"""
-    DEFAULT_DIR = 'metrics'
+    RELATIVE_PATH = 'metric'
diff --git a/qtip/runner/plan.py b/qtip/loader/plan.py
index 265ad8d7..cf517ea2 100644
--- a/qtip/runner/plan.py
+++ b/qtip/loader/plan.py
@@ -7,20 +7,21 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.base.benchmark import Benchmark, Property
-from qtip.runner.suite import Suite
 
+from qtip.base.constant import PropName
+from qtip.loader.base import BaseLoader
+from qtip.loader.qpi import QPISpec
 
-class Plan(Benchmark):
+
+class Plan(BaseLoader):
+    """
+    a benchmark plan is consist of configuration and a QPI list
     """
-    a benchmark plan is consist of basic information and  several suites"""
 
-    DEFAULT_DIR = 'plans'
+    RELATIVE_PATH = 'plan'
 
     def __init__(self, name, paths=None):
-        super(Plan, self).__init__(name, paths=paths)
-        content = self.content()
+        super(Plan, self).__init__(name, paths)
 
-        self.info = content[Property.INFO]
-        self.suites = [Suite(suite, paths=paths)
-                       for suite in content[Property.SUITES]]
+        self.qpis = [QPISpec(qpi, paths=paths)
+                     for qpi in self.content[PropName.QPIS]]
diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py
new file mode 100644
index 00000000..fe2323dd
--- /dev/null
+++ b/qtip/loader/qpi.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from base import BaseLoader
+from metric import MetricSpec
+
+from qtip.base.constant import PropName
+from qtip.util.formula import Formula
+
+
+class QPISpec(BaseLoader):
+    """
+    a QPI specification defines how to calculate a performance index from
+     collected metrics.
+    """
+    RELATIVE_PATH = 'QPI'
+
+    def __init__(self, name, paths=None):
+        super(QPISpec, self).__init__(name, paths=paths)
+        content = self.content
+        self.formula = Formula(content[PropName.FORMULA])
+        self.sections = [Section(record, paths=paths)
+                         for record in content[PropName.SECTIONS]]
+
+
+class Section(object):
+    def __init__(self, content, paths=None):
+        self.name = content[PropName.NAME]
+        self.weight = content[PropName.WEIGHT]
+        self.formula = Formula(content[PropName.FORMULA])
+        self.metrics = [MetricSpec(record, paths=paths)
+                        for record in content[PropName.METRICS]]
diff --git a/qtip/reporter/__init__.py b/qtip/reporter/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/reporter/__init__.py
diff --git a/qtip/reporter/base.py b/qtip/reporter/base.py
new file mode 100644
index 00000000..b931d14d
--- /dev/null
+++ b/qtip/reporter/base.py
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseReporter(object):
+    """benchmark result reporter"""
+    def __init__(self, collector=None):
+        self.collector = collector
diff --git a/qtip/agent/reporter.py b/qtip/reporter/console/__init__.py
index b5c4acfa..02d63288 100644
--- a/qtip/agent/reporter.py
+++ b/qtip/reporter/console/__init__.py
@@ -8,6 +8,8 @@
 ##############################################################################
 
 
-class Reporter(object):
-    """generate test report and push test data to database"""
-    pass
+from qtip.reporter.base import BaseReporter
+
+
+class ConsoleReporter(BaseReporter):
+    """report result to console"""
diff --git a/qtip/reporter/console/timeline.j2 b/qtip/reporter/console/timeline.j2
new file mode 100644
index 00000000..9c18a996
--- /dev/null
+++ b/qtip/reporter/console/timeline.j2
@@ -0,0 +1,8 @@
+{% title %}
+{% for phase in phases %}
+{{ phase.name|upper }}{{ "TIME" }}
+{% for cp in phase.checkpoints %}
+{{ cp.name }}{{ cp.timestamp}}
+{% endfor %}
+{% endfor %}
+Total: {{ total }}
diff --git a/qtip/runner/__init__.py b/qtip/runner/__init__.py
index e69de29b..eab81156 100644
--- a/qtip/runner/__init__.py
+++ b/qtip/runner/__init__.py
@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.constant import PkgName, PropName
+from qtip.base.error import NotFound
+from qtip.collector.stdout import StdoutCollector
+from qtip.driver.random import RandomDriver
+from qtip.reporter.console import ConsoleReporter
+
+
+class Runner(object):
+    def __init__(self, spec, config=None):
+        if config is None:
+            config = spec[PropName.CONFIG]
+
+        driver_name = config[PropName.DRIVER]
+        collector_name = config[PropName.COLLECTOR]
+        reporter_name = config[PropName.REPORTER]
+
+        # TODO(yujunz) dynamically load modules by name
+
+        if driver_name == 'random':
+            self.driver = RandomDriver()
+        else:
+            raise NotFound(driver_name, package=PkgName.DRIVER)
+
+        if collector_name == 'stdout':
+            self.collector = StdoutCollector()
+        else:
+            raise NotFound(collector_name,
+                           package=PkgName.COLLECTOR)
+
+        if reporter_name == 'console':
+            self.reporter = ConsoleReporter()
+        else:
+            raise NotFound(reporter_name,
+                           package=PkgName.REPORTER)
diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py
deleted file mode 100644
index 55033d2b..00000000
--- a/qtip/runner/suite.py
+++ /dev/null
@@ -1,22 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.benchmark import Property
-from qtip.spec.qpi import QPISpec
-from qtip.runner.case import Case
-
-
-class Suite(object):
-    """a suite of benchmark cases under specified condition"""
-    def __init__(self, spec, paths=None):
-        self._paths = paths
-        self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths)
-        self.condition = spec.get(Property.CONDITION, {})
-        self.cases = [Case(case_spec, paths)
-                      for case_spec in spec.get(Property.CASES, [])]
diff --git a/qtip/util/__init__.py b/qtip/util/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/util/__init__.py
diff --git a/qtip/util/formula.py b/qtip/util/formula.py
new file mode 100644
index 00000000..cdfbae86
--- /dev/null
+++ b/qtip/util/formula.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import numpy
+
+from qtip.base.error import make_tbd
+from qtip.base.constant import FormulaName
+
+
+MAPPING = {
+    FormulaName.ARITHMETIC_MEAN: numpy.mean,
+    FormulaName.WEIGHTED_ARITHMETIC_MEAN: numpy.average,
+    # TODO(yujunz) find or implement the method
+    FormulaName.GEOMETRIC_MEAN: make_tbd(FormulaName.GEOMETRIC_MEAN, __name__),
+    # TODO(yujunz) find or implement the method
+    FormulaName.WEIGHTED_GEOMETRIC_MEAN:
+        make_tbd(FormulaName.GEOMETRIC_MEAN, __name__)}
+
+
+class Formula:
+    """calculate a score from give data"""
+    def __init__(self, name):
+        self.calculate = MAPPING[name]
diff --git a/qtip/utils/logger_utils.py b/qtip/util/logger.py
index d5e76a64..d5e76a64 100644
--- a/qtip/utils/logger_utils.py
+++ b/qtip/util/logger.py
diff --git a/requirements.txt b/requirements.txt
index cdaeef70..ec566b83 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,4 @@ Flask==0.11.1
 Flask-RESTful==0.3.5
 flask-restful-swagger==0.19
 ansible==2.1.1.0
+numpy==1.11.3
diff --git a/setup.cfg b/setup.cfg
index 790a365b..ced6d22e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -20,4 +20,7 @@ console_scripts =
 packages =
     qtip
 data_files =
-    etc/qtip = benchmarks/*
+    opt/qtip = opt/*
+
+[tool:pytest]
+addopts = --ignore=legacy
diff --git a/tests/conftest.py b/tests/conftest.py
index 8e2ecf7c..7acb75e6 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,8 +8,11 @@
 ##############################################################################
 
 from os import path
+
 import pytest
 
+from qtip.loader.plan import Plan
+
 
 @pytest.fixture(scope='session')
 def data_root():
@@ -19,3 +22,8 @@ def data_root():
 @pytest.fixture(scope='session')
 def benchmarks_root(data_root):
     return path.join(data_root, 'benchmarks')
+
+
+@pytest.fixture(scope='session')
+def plan(benchmarks_root):
+    return Plan('fake-plan.yaml', [benchmarks_root])
diff --git a/tests/data/benchmarks/QPI/compute.yaml b/tests/data/benchmarks/QPI/compute.yaml
index 5ad8a4b6..e28efaf9 100644
--- a/tests/data/benchmarks/QPI/compute.yaml
+++ b/tests/data/benchmarks/QPI/compute.yaml
@@ -1,44 +1,29 @@
 title: compute
 description: sample performance index of computing
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
 sections:
 - name: Integer
   weight: 0.3
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: dhrystone.yaml
-      workloads:
-        - single_cpu
-        - multi_cpu
+    - dhrystone.yaml
 - name: Float
   weight: 0.3
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: dhrystone.yaml
-      workloads:
-        - single_cpu
-        - multi_cpu
+    - whetstone.yaml
 - name: Memory
   weight: 0.2
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: ramspeed.yaml
-      workloads:
-          - int: [add, average, copy, scale, triad]
-          - float: [add, average, copy, scale, triad]
+    - ramspeed.yaml
 - name: DPI
   weight: 0.1
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: dpi.yaml
-      workloads:
-        - bps
-        - pps
+    - dpi.yaml
 - name: SSL
   weight: 0.1
-  algorithm: geometric mean
+  formula: geometric mean
   metrics:
-    - spec: ssl.yaml
-      workloads:
-        - aes_128_cbc: [512, 1024, 2048, 4096]
-        - rsa_sig: [16, 64, 256, 1024, 8192]
+    - ssl.yaml
diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml
new file mode 100644
index 00000000..aa1097f4
--- /dev/null
+++ b/tests/data/benchmarks/QPI/fake-qpi.yaml
@@ -0,0 +1,9 @@
+name: Fake QPI
+description: a fake QPI producing random result
+formula: weighted arithmetic mean
+sections:
+- name: Fake Section
+  weight: 0.5
+  formula: geometric mean
+  metrics:
+    - fake-metric.yaml
diff --git a/tests/data/benchmarks/metric/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml
new file mode 100644
index 00000000..220b7841
--- /dev/null
+++ b/tests/data/benchmarks/metric/dhrystone.yaml
@@ -0,0 +1,9 @@
+name: dhrystone
+description: >
+  A synthetic computing benchmark program intended to be representative of
+  system (integer) programming.
+links:
+  - https://en.wikipedia.org/wiki/Dhrystone
+workloads:
+  - single_cpu
+  - multi_cpu
diff --git a/tests/data/benchmarks/metrics/dpi.yaml b/tests/data/benchmarks/metric/dpi.yaml
index fc24c8d1..fc24c8d1 100644
--- a/tests/data/benchmarks/metrics/dpi.yaml
+++ b/tests/data/benchmarks/metric/dpi.yaml
diff --git a/tests/data/benchmarks/metric/fake-metric.yaml b/tests/data/benchmarks/metric/fake-metric.yaml
new file mode 100644
index 00000000..c5fcc5c2
--- /dev/null
+++ b/tests/data/benchmarks/metric/fake-metric.yaml
@@ -0,0 +1,7 @@
+name: fake compute
+description: >
+  a fake compute performance index which generate random result
+workloads:  # all supported workloads
+  - fake_workload_1
+  - fake_workload_2
+  - fake_workload_3
diff --git a/tests/data/benchmarks/metrics/ramspeed.yaml b/tests/data/benchmarks/metric/ramspeed.yaml
index bb7618b7..bb7618b7 100644
--- a/tests/data/benchmarks/metrics/ramspeed.yaml
+++ b/tests/data/benchmarks/metric/ramspeed.yaml
diff --git a/tests/data/benchmarks/metrics/ssl.yaml b/tests/data/benchmarks/metric/ssl.yaml
index 21e8add7..21e8add7 100644
--- a/tests/data/benchmarks/metrics/ssl.yaml
+++ b/tests/data/benchmarks/metric/ssl.yaml
diff --git a/tests/data/benchmarks/metric/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml
new file mode 100644
index 00000000..448c9645
--- /dev/null
+++ b/tests/data/benchmarks/metric/whetstone.yaml
@@ -0,0 +1,10 @@
+name: whetstone
+description: >
+  A synthetic benchmark for evaluating the performance of computers.
+  The Whetstone benchmark primarily measures the floating-point arithmetic
+  performance.
+links:
+  - https://en.wikipedia.org/wiki/Whetstone_(benchmark)
+workloads:
+  - single_cpu
+  - multi_cpu
diff --git a/tests/data/benchmarks/metrics/dhrystone.yaml b/tests/data/benchmarks/metrics/dhrystone.yaml
deleted file mode 100644
index b0d55ed2..00000000
--- a/tests/data/benchmarks/metrics/dhrystone.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-name: dhrystone
-description: >
-  a synthetic computing benchmark program intended to be representative of
-  system (integer) programming
-workloads:
-  - single_cpu
-  - multi_cpu
diff --git a/tests/data/benchmarks/metrics/whetstone.yaml b/tests/data/benchmarks/metrics/whetstone.yaml
deleted file mode 100644
index d83680c4..00000000
--- a/tests/data/benchmarks/metrics/whetstone.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-name: dhrystone
-description: a synthetic benchmark for evaluating the performance of computers
-workloads:
-  - single_cpu
-  - multi_cpu
diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml
new file mode 100644
index 00000000..8887f66d
--- /dev/null
+++ b/tests/data/benchmarks/plan/fake-plan.yaml
@@ -0,0 +1,10 @@
+name: fake plan
+description: fake benchmark plan for demonstration and testing
+config:
+  facility: local
+  engineer: local
+  driver: sample
+  collector: logfile
+  reporter: console
+QPIs:
+  - fake-qpi.yaml
diff --git a/tests/data/benchmarks/plans/verification.yaml b/tests/data/benchmarks/plans/verification.yaml
deleted file mode 100644
index b146ee37..00000000
--- a/tests/data/benchmarks/plans/verification.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-title: verification
-description: benchmark plan for QTIP verification
-info:
-  facility: opnfv-ci
-  engineer: opnfv-bot
-suites:
-  - QPI_spec: compute.yaml
-# Uncomment next line to setup suite running condition
-#    condition: {}
-# Uncomment the following lines setup case running configuration
-#    cases:
-#      - metric_spec: dhrystone.yaml
-#        config: {}
-#      - metric_spec: whetstone.yaml
-#        config: {}
-#      - metric_spec: ramspeed.yaml
-#        config: {}
-#      - metric_spec: dpi.yaml
-#        config: {}
-#      - metric_spec: ssl.yaml
-#        config: {}
diff --git a/tests/unit/runner/case_test.py b/tests/unit/cli/test_metric.py
index 59a54a84..ed10689c 100644
--- a/tests/unit/runner/case_test.py
+++ b/tests/unit/cli/test_metric.py
@@ -7,9 +7,22 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from qtip.spec.metric import MetricSpec
+import pytest
+from click.testing import CliRunner
 
+from qtip.cli.commands.cmd_metric import cli
 
-def init_test(case):
-    assert isinstance(case.metric_spec, MetricSpec)
-    assert isinstance(case.config, dict)
+
+@pytest.fixture()
+def runner():
+    return CliRunner()
+
+
+def test_list(runner):
+    result = runner.invoke(cli, ['list'])
+    assert result.output == ''
+
+
+def test_run(runner):
+    result = runner.invoke(cli, ['run'])
+    assert result.output == ''
diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py
index 5eced700..d2be0388 100644
--- a/tests/unit/loader/metric_test.py
+++ b/tests/unit/loader/metric_test.py
@@ -9,8 +9,8 @@
 
 import pytest
 
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
+from qtip.base.constant import PropName
+from qtip.loader.metric import MetricSpec
 
 
 @pytest.fixture(scope='module')
@@ -29,17 +29,17 @@ def init_test(metric_spec):
 
 def list_all_test():
     metric_list = MetricSpec.list_all()
-    assert len(list(metric_list)) is 1
+    assert len(list(metric_list)) is 6
     for desc in metric_list:
-        assert Property.NAME in desc
-        assert Property.DESCRIPTION in desc
-        assert Property.ABSPATH in desc
-        assert Property.ABSPATH is not None
+        assert PropName.NAME in desc
+        assert PropName.DESCRIPTION in desc
+        assert PropName.ABSPATH in desc
+        assert PropName.ABSPATH is not None
 
 
 def content_test(metric):
-    content = metric.content()
-    assert Property.NAME in content
-    assert Property.DESCRIPTION in content
-    assert Property.WORKLOADS in content
-    assert isinstance(content[Property.WORKLOADS], list)
+    content = metric.content
+    assert PropName.NAME in content
+    assert PropName.DESCRIPTION in content
+    assert PropName.WORKLOADS in content
+    assert isinstance(content[PropName.WORKLOADS], list)
diff --git a/tests/unit/runner/plan_test.py b/tests/unit/loader/plan_test.py
index d783e5e9..6aab5e8a 100644
--- a/tests/unit/runner/plan_test.py
+++ b/tests/unit/loader/plan_test.py
@@ -9,12 +9,15 @@
 
 import pytest
 
-from qtip.base.benchmark import Property
-from qtip.runner.plan import Plan
+from qtip.base.constant import PropName
+from qtip.loader.plan import Plan, QPISpec
 
 
 def test_init(plan):
-    assert plan.name == 'verification'
+    assert plan.name == 'fake plan'
+    assert isinstance(plan.content, dict)
+    for qpi in plan.qpis:
+        assert isinstance(qpi, QPISpec)
 
     with pytest.raises(TypeError) as excinfo:
         Plan()
@@ -26,13 +29,15 @@ def test_list_all(benchmarks_root):
     plan_list = Plan.list_all(paths=[benchmarks_root])
     assert len(list(plan_list)) is 1
     for desc in plan_list:
-        assert Property.NAME in desc
-        assert Property.CONTENT in desc
-        assert Property.ABSPATH in desc
-        assert Property.ABSPATH is not None
+        assert PropName.NAME in desc
+        assert PropName.CONTENT in desc
+        assert PropName.ABSPATH in desc
+        assert PropName.ABSPATH is not None
 
 
 def test_content(plan):
-    content = plan.content()
-    assert Property.TITLE in content
-    assert Property.DESCRIPTION in content
+    content = plan.content
+    assert PropName.NAME in content
+    assert PropName.DESCRIPTION in content
+    assert PropName.CONFIG in content
+    assert PropName.QPIS in content
diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py
index bfa1f580..4b3fd4d0 100644
--- a/tests/unit/loader/qpi_test.py
+++ b/tests/unit/loader/qpi_test.py
@@ -9,8 +9,8 @@
 
 import pytest
 
-from qtip.base.benchmark import Algorithm, Property
-from qtip.spec.qpi import QPISpec
+from qtip.base.constant import FormulaName, PropName
+from qtip.loader.qpi import QPISpec
 
 QPI_SPEC = 'compute.yaml'
 
@@ -31,22 +31,22 @@ def test_init(qpi_spec):
 
 def test_list_all(benchmarks_root):
     qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root])
-    assert len(list(qpi_spec_list)) is 1
+    assert len(list(qpi_spec_list)) is 2
     for item in qpi_spec_list:
-        assert Property.NAME in item
-        assert Property.CONTENT in item
-        assert Property.ABSPATH in item
-        assert Property.ABSPATH is not None
+        assert PropName.NAME in item
+        assert PropName.CONTENT in item
+        assert PropName.ABSPATH in item
+        assert PropName.ABSPATH is not None
 
 
 def test_content(qpi_spec):
-    content = qpi_spec.content()
-    assert Property.DESCRIPTION in content
-    assert Property.ALGORITHM in content
-    assert Property.SECTIONS in content
+    content = qpi_spec.content
+    assert PropName.DESCRIPTION in content
+    assert PropName.FORMULA in content
+    assert PropName.SECTIONS in content
 
-    assert content[Property.ALGORITHM] in Algorithm.__dict__.values()
-    sections = content[Property.SECTIONS]
+    assert content[PropName.FORMULA] in FormulaName.__dict__.values()
+    sections = content[PropName.SECTIONS]
     assert isinstance(sections, list)
     for section in sections:
-        assert Property.NAME in section
+        assert PropName.NAME in section
diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py
deleted file mode 100644
index 6d14f7ae..00000000
--- a/tests/unit/runner/conftest.py
+++ /dev/null
@@ -1,30 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.base.benchmark import Property
-from qtip.runner.case import Case
-from qtip.runner.plan import Plan
-from qtip.runner.suite import Suite
-
-
-@pytest.fixture(scope='module')
-def plan(benchmarks_root):
-    return Plan('verification.yaml', paths=[benchmarks_root])
-
-
-@pytest.fixture(scope='module')
-def suite(plan):
-    return Suite(plan[Property.SUITES][0])
-
-
-@pytest.fixture(scope='module')
-def case(suite):
-    return Case(suite[Property.CASES][0])
diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py
deleted file mode 100644
index 7dad8f62..00000000
--- a/tests/unit/runner/suite_test.py
+++ /dev/null
@@ -1,27 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.runner.suite import Suite
-from qtip.runner.case import Case
-from qtip.spec.qpi import QPISpec
-
-
-def init_test(suite):
-    assert isinstance(suite.qpi, QPISpec)
-    assert isinstance(suite.condition, dict)
-    assert isinstance(suite.cases, list)
-    for case in suite.cases:
-        assert isinstance(case, Case)
-
-    with pytest.raises(TypeError) as excinfo:
-        Suite()
-    assert '__init__() takes exactly 2 arguments (1 given)' \
-           in str(excinfo.value)
diff --git a/tox.ini b/tox.ini
index 7d17af65..03283e42 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27
+envlist = py27,pep8
 skipsdist = True
 
 [testenv]
@@ -32,4 +32,4 @@ commands = flake8 {toxinidir}
 show-source = True
 ignore = E123,E125,H803,E501
 builtins = _
-exclude = build,dist,doc,.eggs,.git,.tox,.venv
+exclude = build,dist,doc,legacy,.eggs,.git,.tox,.venv