Merge "Deprecate plan and metric loader"

23 files changed, 1 insertions, 891 deletions

diff --git a/benchmarks/metric/dhrystone.yaml b/benchmarks/metric/dhrystone.yaml
deleted file mode 100644
index e4d26248..00000000
--- a/benchmarks/metric/dhrystone.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-name: dhrystone
-description: >
-  A synthetic computing benchmark program intended to be representative of
-  system (integer) programming.
-links:
-  - https://en.wikipedia.org/wiki/Dhrystone
-workloads:
-  - single_cpu
-  - multi_cpu
diff --git a/benchmarks/metric/dpi.yaml b/benchmarks/metric/dpi.yaml
deleted file mode 100644
index c90075d2..00000000
--- a/benchmarks/metric/dpi.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-name: dpi
-description: deep packet inspection
-workloads:
-  - bps
-  - pps
diff --git a/benchmarks/metric/ramspeed.yaml b/benchmarks/metric/ramspeed.yaml
deleted file mode 100644
index e2e2b98f..00000000
--- a/benchmarks/metric/ramspeed.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-name: ramspeed
-description: a memory performance benchmarking
-workloads:
-    - int: [add, average, copy, scale, triad]
-    - float: [add, average, copy, scale, triad]
diff --git a/benchmarks/metric/ssl.yaml b/benchmarks/metric/ssl.yaml
deleted file mode 100644
index 2b01dccd..00000000
--- a/benchmarks/metric/ssl.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-name: ssl
-description: a Secure Sockets Layer performance benchmarking
-workloads:
-  - aes_128_cbc: [512, 1024, 2048, 4096]
-  - rsa_sig: [16, 64, 256, 1024, 8192]
diff --git a/benchmarks/metric/whetstone.yaml b/benchmarks/metric/whetstone.yaml
deleted file mode 100644
index 625ea87f..00000000
--- a/benchmarks/metric/whetstone.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-name: whetstone
-description: >
-  A synthetic benchmark for evaluating the performance of computers.
-  The Whetstone benchmark primarily measures the floating-point arithmetic
-  performance.
-links:
-  - https://en.wikipedia.org/wiki/Whetstone_(benchmark)
-workloads:
-  - single_cpu
-  - multi_cpu
diff --git a/benchmarks/plan/compute.yaml b/benchmarks/plan/compute.yaml
deleted file mode 100644
index f4a7a2dc..00000000
--- a/benchmarks/plan/compute.yaml
+++ /dev/null
@@ -1,120 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-name: compute QPI
-description: compute QPI profile
-info:
-  facility: local
-  engineer: local
-config:
-  driver: ansible
-  collectors:
-    - type: logfile
-      paths:
-        - '../../external/dpi/'
-      logs:
-        - filename: dpi_dump.txt
-          parsers:
-            - type: grep
-              regex: |-
-                ^\s+nDPI throughput:.+?(?P<pps>\d+.\d+)\sM\spps.+
-                ?(?P<bps>\d+.\d+)\sGb\/sec
-    - type: logfile
-      paths:
-        - '../../external/ramspeed/'
-      logs:
-        - filename: Intmem
-          parsers:
-            - type: grep
-              regex: '^INTEGER\s+BatchRun\s+Copy:\s+?(?P<integer_copy>\d+\.\d+)\sMB/s$'
-            - type: grep
-              regex: '^INTEGER\s+BatchRun\s+Scale:\s+?(?P<integer_scale>\d+\.\d+)\sMB/s$'
-            - type: grep
-              regex: '^INTEGER\s+BatchRun\s+Add:\s+?(?P<integer_add>\d+\.\d+)\sMB/s$'
-            - type: grep
-              regex: '^INTEGER\s+BatchRun\s+Triad:\s+?(?P<integer_triad>\d+\.\d+)\sMB/s$'
-            - type: grep
-              regex: '^INTEGER\s+BatchRun\s+AVERAGE:\s+?(?P<integer_average>\d+\.\d+)\sMB/s$'
-        - filename: Floatmem
-          parsers:
-            - type: grep
-              regex: '^FL-POINT\s+BatchRun\s+Copy:\s+?(?P<float_copy>\d+\.\d+)\sMB/s$'
-            - type: grep
-              regex: '^FL-POINT\s+BatchRun\s+Scale:\s+?(?P<float_scale>\d+\.\d+)\sMB/s$'
-            - type: grep
-              regex: '^FL-POINT\s+BatchRun\s+Add:\s+?(?P<float_add>\d+\.\d+)\sMB/s$'
-            - type: grep
-              regex: '^FL-POINT\s+BatchRun\s+Triad:\s+?(?P<float_triad>\d+\.\d+)\sMB/s$'
-            - type: grep
-              regex: '^FL-POINT\s+BatchRun\s+AVERAGE:\s+?(?P<float_average>\d+\.\d+)\sMB/s$'
-    - type: logfile
-      paths:
-        - '../../external/ssl/'
-      logs:
-        - filename: RSA_dump
-          parsers:
-            - type: grep
-              regex: |-
-                ^rsa\s+512\sbits\s.+
-                ?(?P<rsa_sign_512>\d+\.\d)\s+
-                ?(?P<rsa_verify_512>\d+\.\d)$
-            - type: grep
-              regex: |-
-                ^rsa\s+1024\sbits\s.+
-                ?(?P<rsa_sign_1024>\d+\.\d)\s+
-                ?(?P<rsa_verify_1024>\d+\.\d)$
-            - type: grep
-              regex: |-
-                ^rsa\s+2048\sbits\s.+
-                ?(?P<rsa_sign_2048>\d+\.\d)\s+
-                ?(?P<rsa_verify_2048>\d+\.\d)$
-            - type: grep
-              regex: |-
-                ^rsa\s+4096\sbits\s.+
-                ?(?P<rsa_sign_4096>\d+\.\d)\s+
-                ?(?P<rsa_verify_4096>\d+\.\d)$
-        - filename: AES-128-CBC_dump
-          parsers:
-            - type: grep
-              regex: |-
-                ^aes-128-cbc\s+
-                ?(?P<aes_128_cbc_16_bytes>\d+\.\w+)\s+
-                ?(?P<aes_128_cbc_64_bytes>\d+\.\w+)\s+
-                ?(?P<aes_128_cbc_256_bytes>\d+\.\w+)\s+
-                ?(?P<aes_128_cbc_1024_bytes>\d+\.\w+)\s+
-                ?(?P<aes_128_cbc_8192_bytes>\d+\.\w+)$
-    - type: logfile
-      paths:
-        - '../../external/sysinfo'
-      logs:
-        - filename: top.log
-          parsers:
-            - type: grep
-              regex: 'Cpu\(s\):.+?(?P<cpu_idle>\d+\.\d)\sid'
-        - filename: inxi.log
-          parsers:
-            - type: grep
-              regex: '.+\s+Host:\s+(?P<hostname>.+)\sKernel'
-            - type: grep
-              regex: '.+\sMemory:\s+(?P<memory>.+MB)\s'
-            - type: grep
-              regex: '^CPU\(s\):\s+(?P<cpu>.+)'
-            - type: grep
-              regex: '.+\sDistro:\s+(?P<os>.+)'
-            - type: grep
-              regex: '.+\sKernel:\s+(?P<kernel>.+)\sConsole'
-            - type: grep
-              regex: '.+\s+HDD Total Size:\s+(?P<disk>.+)\s'
-            - type: grep
-              regex: '.+\sproduct:\s+(?P<product>.+)\sversion'
-  reporter:
-    name: console
-    # transform collected data into timeline
-    transformer: timeline
-QPIs:
-  - compute.yaml
diff --git a/benchmarks/plan/sample.yaml b/benchmarks/plan/sample.yaml
deleted file mode 100644
index 04e8caf9..00000000
--- a/benchmarks/plan/sample.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-name: sample
-description: sample benchmark plan for testing default path
-config:
-  collectors: []
-  reporters: []
-QPIs: []
diff --git a/qtip/api/controllers/metric.py b/qtip/api/controllers/metric.py
deleted file mode 100644
index 96cd985c..00000000
--- a/qtip/api/controllers/metric.py
+++ /dev/null
@@ -1,25 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import httplib
-
-from qtip.api.controllers import common
-from qtip.loader import metric
-
-
-def list_metrics():
-    metrics = list(metric.MetricSpec.list_all())
-    metrics_by_name = [m['name'] for m in metrics]
-    return {'metrics': metrics_by_name}, httplib.OK
-
-
-@common.check_endpoint_for_error(resource='Metric')
-def get_metric(name):
-    metric_spec = metric.MetricSpec(name)
-    return metric_spec.content
diff --git a/qtip/api/controllers/plan.py b/qtip/api/controllers/plan.py
deleted file mode 100644
index 00593878..00000000
--- a/qtip/api/controllers/plan.py
+++ /dev/null
@@ -1,31 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import httplib
-
-from qtip.api.controllers import common
-from qtip.base import error
-from qtip.loader import plan
-
-
-def list_plans():
-    plans = list(plan.Plan.list_all())
-    plans_by_name = [p['name'] for p in plans]
-    return {'plans': plans_by_name}, httplib.OK
-
-
-@common.check_endpoint_for_error(resource='Plan')
-def get_plan(name):
-    plan_spec = plan.Plan(name)
-    return plan_spec.content
-
-
-@common.check_endpoint_for_error(resource='Plan', operation='Run')
-def run_plan(name, action="run"):
-    raise error.ToBeDoneError('run_plan', 'plan')
diff --git a/qtip/api/swagger/swagger.yaml b/qtip/api/swagger/swagger.yaml
index 51c3ebb8..8cb0e23b 100644
--- a/qtip/api/swagger/swagger.yaml
+++ b/qtip/api/swagger/swagger.yaml
@@ -16,95 +16,6 @@ consumes:
 produces:
   - application/json
 paths:
-  /plans:
-    get:
-      summary: List all plans
-      operationId: qtip.api.controllers.plan.list_plans
-      tags:
-        - Plan
-        - Standalone
-      responses:
-        200:
-          description: A list of plans
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/Plans'
-        501:
-          description: Resource not implemented
-          schema:
-            $ref: '#/definitions/Error'
-        default:
-          description: Unexpected error
-          schema:
-            $ref: '#/definitions/Error'
-  /plans/{name}:
-    get:
-      summary: Get a plan by plan name
-      operationId: qtip.api.controllers.plan.get_plan
-      tags:
-        - Plan
-        - Standalone
-      parameters:
-        - name: name
-          in: path
-          description: Plan name
-          required: true
-          type: string
-      responses:
-        200:
-          description: Plan information
-          schema:
-            $ref: '#/definitions/Plan'
-        404:
-          description: Plan not found
-          schema:
-            $ref: '#/definitions/Error'
-        501:
-          description: Resource not implemented
-          schema:
-            $ref: '#/definitions/Error'
-        default:
-          description: Unexpected error
-          schema:
-            $ref: '#/definitions/Error'
-    post:
-      summary: Run a plan and return results
-      operationId: qtip.api.controllers.plan.run_plan
-      tags:
-        - Plan
-        - Standalone
-      parameters:
-        - name: name
-          in: path
-          description: Plan name
-          required: true
-          type: string
-        - name: action
-          in: query
-          description: action for a plan
-          required: true
-          type: string
-      responses:
-        200:
-          description: Result of the run of the plan
-          #TODO (akhil) define schema
-        404:
-          description: Plan not found
-          schema:
-            $ref: '#/definitions/Error'
-        400:
-          description: Invalid parameters
-          schema:
-            $ref: '#/definitions/Error'
-        501:
-          description: Resource not implemented
-          schema:
-            $ref: '#/definitions/Error'
-        default:
-          description: Unexpected error
-          schema:
-            $ref: '#/definitions/Error'
   /qpis:
     get:
       summary: List all QPIs
@@ -158,112 +69,7 @@ paths:
           description: Unexpected error
           schema:
             $ref: '#/definitions/Error'
-  /metrics:
-    get:
-      summary: List all metrics
-      operationId: qtip.api.controllers.metric.list_metrics
-      tags:
-        - Metric
-        - Standalone
-        - Agent
-      responses:
-        200:
-          description: A list of metrics
-          schema:
-            items:
-              $ref: '#/definitions/Metrics'
-        501:
-          description: Resource not implemented
-          schema:
-            $ref: '#/definitions/Error'
-        default:
-          description: Unexpected error
-          schema:
-            $ref: '#/definitions/Error'
-  /metrics/{name}:
-    get:
-      summary: Get a metric
-      operationId: qtip.api.controllers.metric.get_metric
-      tags:
-        - Metric
-        - Standalone
-        - Agent
-      parameters:
-        - name: name
-          in: path
-          description: Metric name
-          required: true
-          type: string
-      responses:
-        200:
-          description: Metric information
-          schema:
-            $ref: '#/definitions/Metric'
-        404:
-          description: Metric not found
-          schema:
-            $ref: '#/definitions/Error'
-        501:
-          description: Resource not implemented
-          schema:
-            $ref: '#/definitions/Error'
-        default:
-          description: Unexpected error
-          schema:
-            $ref: '#/definitions/Error'
 definitions:
-  Plan:
-    type: object
-    required:
-      - name
-    properties:
-      name:
-        type: string
-      description:
-        type: string
-      info:
-        type: object
-      config:
-        type: object
-      QPIs:
-        type: array
-        items:
-          type: object
-  Plans:
-    type: object
-    required:
-      - plans
-    properties:
-      plans:
-        type: array
-        items:
-          type: string
-  Metric:
-    type: object
-    required:
-      - name
-    properties:
-      name:
-        type: string
-      description:
-        type: string
-      links:
-        type: array
-        items:
-          type: string
-      workloads:
-        type: array
-        items:
-          type: string
-  Metrics:
-    type: object
-    required:
-      - metrics
-    properties:
-      metrics:
-        type: array
-        items:
-          type: string
   QPI:
     type: object
     required:
diff --git a/qtip/cli/commands/cmd_metric.py b/qtip/cli/commands/cmd_metric.py
deleted file mode 100644
index 0a385898..00000000
--- a/qtip/cli/commands/cmd_metric.py
+++ /dev/null
@@ -1,54 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 taseer94@gmail.com and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import click
-from colorama import Fore
-import os
-
-from qtip.base.error import InvalidContentError
-from qtip.base.error import NotFoundError
-from qtip.cli import utils
-from qtip.loader.metric import MetricSpec
-
-
-@click.group()
-def cli():
-    ''' Performance Metrics Group '''
-    pass
-
-
-@cli.command('list', help='List all the Metric Groups')
-def cmd_list():
-    metrics = MetricSpec.list_all()
-    table = utils.table('Metrics', metrics)
-    click.echo(table)
-
-
-@cli.command('show', help='View details of a Metric')
-@click.argument('name')
-def show(name):
-    try:
-        metric = MetricSpec('{}.yaml'.format(name))
-    except NotFoundError as nf:
-        click.echo(Fore.RED + "ERROR: metric spec: " + nf.message)
-    except InvalidContentError as ice:
-        click.echo(Fore.RED + "ERROR: metric spec " + ice.message)
-    else:
-        cnt = metric.content
-        output = utils.render('metric', cnt)
-        click.echo(output)
-
-
-@cli.command('run', help='Run performance test')
-@click.argument('name')
-@click.option('-p', '--path', help='Path to store results')
-def run(name, path):
-    runner_path = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir,
-                               'runner/runner.py')
-    os.system('python {0} -b {1} -d {2}'.format(runner_path, name, path))
diff --git a/qtip/loader/metric.py b/qtip/loader/metric.py
deleted file mode 100644
index 842fcdbf..00000000
--- a/qtip/loader/metric.py
+++ /dev/null
@@ -1,16 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from yaml_file import YamlFileLoader
-
-
-class MetricSpec(YamlFileLoader):
-    """metrics in QTIP are categorized by performance test tools, such as
-    dhrystone, whetstone and etc"""
-    RELATIVE_PATH = 'metric'
diff --git a/qtip/loader/plan.py b/qtip/loader/plan.py
deleted file mode 100644
index e15651a3..00000000
--- a/qtip/loader/plan.py
+++ /dev/null
@@ -1,58 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from qtip.base.constant import BaseProp
-from qtip.collector import CollectorProp as CProp
-from qtip.collector.logfile import LogfileCollector
-from qtip.loader.yaml_file import YamlFileLoader
-from qtip.loader.qpi import QPISpec
-
-
-# TODO(yujunz) more elegant way to load module dynamically
-def load_collector(type_name):
-    if type_name == LogfileCollector.TYPE:
-        return LogfileCollector
-    else:
-        raise Exception("Invalid collector type: {}".format(type_name))
-
-
-class Plan(YamlFileLoader):
-    """
-    a benchmark plan is consist of configuration and a QPI list
-    """
-
-    RELATIVE_PATH = 'plan'
-
-    def __init__(self, name, paths=None):
-        super(Plan, self).__init__(name, paths)
-
-        _config = self.content[PlanProp.CONFIG]
-
-        self.collectors = [load_collector(c[CProp.TYPE])(c, self)
-                           for c in _config[PlanProp.COLLECTORS]]
-
-        self.qpis = [QPISpec(qpi, paths=paths)
-                     for qpi in self.content[PlanProp.QPIS]]
-
-
-class PlanProp(BaseProp):
-    # plan
-    INFO = 'info'
-
-    FACILITY = 'facility'
-    ENGINEER = 'engineer'
-
-    CONFIG = 'config'
-
-    DRIVER = 'driver'
-    COLLECTORS = 'collectors'
-    REPORTER = 'reporter'
-
-    QPIS = 'QPIs'
diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py
index 73da61e9..2a85766e 100644
--- a/qtip/loader/qpi.py
+++ b/qtip/loader/qpi.py
@@ -8,7 +8,6 @@
 ##############################################################################
 
 from yaml_file import YamlFileLoader
-from metric import MetricSpec
 
 from qtip.base.constant import SpecProp
 from qtip.util.formula import Formula
@@ -34,5 +33,3 @@ class Section(object):
         self.name = content[SpecProp.NAME]
         self.weight = content[SpecProp.WEIGHT]
         self.formula = Formula(content[SpecProp.FORMULA])
-        self.metrics = [MetricSpec(record, paths=paths)
-                        for record in content[SpecProp.METRICS]]
diff --git a/resources/ansible_roles/inxi/tasks/main.yml b/resources/ansible_roles/inxi/tasks/main.yml
index c1f0b28c..af0f39e1 100644
--- a/resources/ansible_roles/inxi/tasks/main.yml
+++ b/resources/ansible_roles/inxi/tasks/main.yml
@@ -1,4 +1,4 @@
-##############################################################################
+  ##############################################################################
 # Copyright (c) 2017 ZTE Corporation and others.
 #
 # All rights reserved. This program and the accompanying materials
diff --git a/tests/conftest.py b/tests/conftest.py
index ddec94b8..8e812d66 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,9 +11,6 @@ from os import path
 
 import pytest
 
-from qtip.loader.plan import Plan
-from qtip.loader.plan import PlanProp
-
 
 @pytest.fixture(scope='session')
 def data_root():
@@ -26,26 +23,6 @@ def benchmarks_root(data_root):
 
 
 @pytest.fixture(scope='session')
-def plan(benchmarks_root):
-    return Plan('doctor.yaml', [benchmarks_root])
-
-
-@pytest.fixture(scope='session')
-def plan_config(plan):
-    return plan.content[PlanProp.CONFIG]
-
-
-@pytest.fixture(scope='session')
-def collectors_config(plan_config):
-    return plan_config[PlanProp.COLLECTORS]
-
-
-@pytest.fixture(scope='session')
-def logfile_config(collectors_config):
-    return collectors_config[0]
-
-
-@pytest.fixture(scope='session')
 def metrics():
     return {
         "ssl_rsa": {
diff --git a/tests/unit/api/metric_controller_test.py b/tests/unit/api/metric_controller_test.py
deleted file mode 100644
index caba7972..00000000
--- a/tests/unit/api/metric_controller_test.py
+++ /dev/null
@@ -1,37 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import httplib
-import json
-
-from qtip.base.constant import BaseProp
-
-
-def test_get_list_metrics(app_client):
-    response_success = app_client.get("/v1.0/metrics")
-    assert response_success.status_code == httplib.OK
-    metric_list = json.loads(response_success.data)['metrics']
-    assert len(metric_list) > 0
-    assert metric_list[0].endswith('.yaml')
-
-
-def test_get_metric(app_client):
-    response_success = app_client.get("/v1.0/metrics/dpi.yaml")
-    assert response_success.status_code == httplib.OK
-    metric_data = json.loads(response_success.data)
-    assert BaseProp.NAME in metric_data
-    assert BaseProp.WORKLOADS in metric_data
-    assert isinstance(metric_data[BaseProp.WORKLOADS], list)
-
-
-def test_get_metric_not_found(app_client):
-    response_not_found = app_client.get("/v1.0/metrics/fake.yaml")
-    response_data = json.loads(response_not_found.data)
-    assert response_not_found.status_code == httplib.NOT_FOUND
-    assert response_data['title'] == "Metric not found"
diff --git a/tests/unit/api/plan_controller_test.py b/tests/unit/api/plan_controller_test.py
deleted file mode 100644
index 136bd3c6..00000000
--- a/tests/unit/api/plan_controller_test.py
+++ /dev/null
@@ -1,49 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import httplib
-import json
-
-
-from qtip.loader.plan import PlanProp
-
-
-def test_invalid_url(app_client):
-    response_url_not_found = app_client.get("/v1.0/fakeresource")
-    assert response_url_not_found.status_code == httplib.NOT_FOUND
-
-
-def test_get_list_plans(app_client):
-    response_success = app_client.get("/v1.0/plans")
-    assert response_success.status_code == httplib.OK
-    plan_list = json.loads(response_success.data)['plans']
-    assert len(plan_list) > 0
-    assert plan_list[0].endswith('.yaml')
-
-
-def test_get_plan(app_client):
-    response_success = app_client.get("/v1.0/plans/sample.yaml")
-    assert response_success.status_code == httplib.OK
-    plan_data = json.loads(response_success.data)
-    assert PlanProp.NAME in plan_data
-    assert PlanProp.DESCRIPTION in plan_data
-    assert PlanProp.CONFIG in plan_data
-    assert PlanProp.QPIS in plan_data
-
-
-def test_get_plan_not_found(app_client):
-    response_not_found = app_client.get("/v1.0/plans/fake.yaml")
-    response_data = json.loads(response_not_found.data)
-    assert response_not_found.status_code == httplib.NOT_FOUND
-    assert response_data['title'] == "Plan not found"
-
-
-def test_runner_not_implemented(app_client):
-    response_error = app_client.post("/v1.0/plans/fake.yaml?action=run", follow_redirects=False)
-    assert response_error.status_code == httplib.NOT_IMPLEMENTED
diff --git a/tests/unit/cli/cmd_metric_test.py b/tests/unit/cli/cmd_metric_test.py
deleted file mode 100644
index c92e944b..00000000
--- a/tests/unit/cli/cmd_metric_test.py
+++ /dev/null
@@ -1,46 +0,0 @@
-###############################################################
-# Copyright (c) 2017 taseer94@gmail.com and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from click.testing import CliRunner
-from qtip.cli.entry import cli
-
-
-@pytest.fixture(scope="module")
-def runner():
-    return CliRunner()
-
-
-def test_list(runner):
-    result = runner.invoke(cli, ['metric', 'list'])
-    assert 'dhrystone' and 'whetstone' and 'dpi' and \
-           'ramspeed' and 'fake-metric' and 'ssl' \
-           in result.output
-
-
-def test_run(runner):
-    result = runner.invoke(cli, ['metric', 'run', 'fake-metric'])
-    assert result.output == ''
-
-    result = runner.invoke(cli, ['metric', 'run'])
-    assert 'Missing argument "name".' in result.output
-
-
-def test_show(runner):
-    result = runner.invoke(cli, ['metric', 'show', 'dhrystone'])
-    assert 'Name: dhrystone' in result.output
-    assert 'Description: A synthetic computing benchmark program intended to be representative of' \
-           'system (integer) programming.'
-
-    result = runner.invoke(cli, ['metric', 'show'])
-    assert 'Missing argument "name".' in result.output
-
-    result = runner.invoke(cli, ['metric', 'show', 'xyz'])
-    assert "ERROR: metric spec: xyz not found" in result.output
diff --git a/tests/unit/collector/collector_test.py b/tests/unit/collector/collector_test.py
deleted file mode 100644
index 17fe1af1..00000000
--- a/tests/unit/collector/collector_test.py
+++ /dev/null
@@ -1,18 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from qtip.loader.plan import load_collector
-from qtip.collector import CollectorProp as CProp
-
-
-def test_load_collector(collectors_config):
-    for c in collectors_config:
-        collector = load_collector(c[CProp.TYPE])
-        assert collector.TYPE == c[CProp.TYPE]
diff --git a/tests/unit/collector/logfile_test.py b/tests/unit/collector/logfile_test.py
deleted file mode 100644
index a76aa3ee..00000000
--- a/tests/unit/collector/logfile_test.py
+++ /dev/null
@@ -1,33 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.collector.logfile import LogfileCollector
-
-
-@pytest.fixture
-def logfile_collector(logfile_config, plan):
-    return LogfileCollector(logfile_config, plan)
-
-
-def test_run(logfile_collector):
-    collected = logfile_collector.run()
-    assert collected['groupdict'] == {
-        'event_posted': '1482894965.3',
-        'host_down': '1482894965.51',
-        'network_down': '1482894965.164096803',
-        'notified': '1482894965.63',
-        'vm_error': '1482894965.3'
-    }
-    assert list(collected['groups']) == ['1482894965.63',
-                                         '1482894965.3',
-                                         '1482894965.3',
-                                         '1482894965.51',
-                                         '1482894965.164096803']
diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py
deleted file mode 100644
index 619d5e00..00000000
--- a/tests/unit/loader/metric_test.py
+++ /dev/null
@@ -1,43 +0,0 @@
-###############################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.base.constant import BaseProp
-from qtip.loader.metric import MetricSpec
-
-
-@pytest.fixture(scope='module')
-def metric_spec(benchmarks_root):
-    return MetricSpec('dhrystone.yaml', paths=[benchmarks_root])
-
-
-def init_test(metric_spec):
-    assert metric_spec.name == 'dhrystone'
-
-    with pytest.raises(TypeError) as excinfo:
-        MetricSpec()
-    assert '__init__() takes at least 2 arguments (1 given)' \
-           in str(excinfo.value)
-
-
-def list_all_test(benchmarks_root):
-    metric_list = list(MetricSpec.list_all(paths=[benchmarks_root]))
-    assert len(metric_list) is 6
-    for desc in metric_list:
-        assert BaseProp.NAME in desc
-        assert BaseProp.ABSPATH in desc
-        assert BaseProp.ABSPATH is not None
-
-
-def content_test(metric_spec):
-    content = metric_spec.content
-    assert BaseProp.NAME in content
-    assert BaseProp.WORKLOADS in content
-    assert isinstance(content[BaseProp.WORKLOADS], list)
diff --git a/tests/unit/loader/plan_test.py b/tests/unit/loader/plan_test.py
deleted file mode 100644
index 4c92e8d5..00000000
--- a/tests/unit/loader/plan_test.py
+++ /dev/null
@@ -1,52 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.collector.logfile import LogfileCollector
-from qtip.loader.plan import load_collector
-from qtip.loader.plan import Plan
-from qtip.loader.plan import PlanProp
-
-
-def test_construct(benchmarks_root):
-    sample = Plan('sample.yaml')
-    assert isinstance(sample, Plan)
-
-    # fixture can not be used in pytest.mark.parametrized
-    sample = Plan('sample.yaml', [benchmarks_root])
-    assert isinstance(sample, Plan)
-
-
-def test_invalid_construct():
-    with pytest.raises(TypeError) as excinfo:
-        Plan()
-    assert '__init__() takes at least 2 arguments (1 given)' \
-           in str(excinfo.value)
-
-
-def test_list_all(benchmarks_root):
-    plan_list = list(Plan.list_all(paths=[benchmarks_root]))
-    assert len(plan_list) is 3
-    for desc in plan_list:
-        assert PlanProp.NAME in desc
-        assert PlanProp.ABSPATH in desc
-        assert PlanProp.ABSPATH is not None
-
-
-def test_content(plan):
-    content = plan.content
-    assert PlanProp.NAME in content
-    assert PlanProp.DESCRIPTION in content
-    assert PlanProp.CONFIG in content
-    assert PlanProp.QPIS in content
-
-
-def test_load_collector():
-    assert load_collector(LogfileCollector.TYPE) is LogfileCollector