summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.gitignore7
-rw-r--r--CONTRIBUTING.md22
-rw-r--r--DEVELOP.md101
-rw-r--r--docker/Dockerfile47
-rw-r--r--docs/designspec/index.rst1
-rw-r--r--docs/designspec/integration_with_yardstick.rst92
-rw-r--r--legacy/DO-NOT-DELETE2
-rw-r--r--legacy/__init__.py (renamed from qtip/agent/__init__.py)0
-rw-r--r--legacy/api/__init__.py (renamed from qtip/api/handler/__init__.py)0
-rw-r--r--legacy/api/handler/__init__.py (renamed from qtip/api/model/__init__.py)0
-rw-r--r--legacy/api/handler/db.py (renamed from qtip/api/handler/db.py)0
-rw-r--r--legacy/api/handler/job_handler.py (renamed from qtip/api/handler/job_handler.py)6
-rw-r--r--legacy/api/handler/result_handler.py (renamed from qtip/api/handler/result_handler.py)0
-rw-r--r--legacy/api/model/__init__.py (renamed from qtip/api/router/__init__.py)0
-rw-r--r--legacy/api/model/job_model.py (renamed from qtip/api/model/job_model.py)0
-rw-r--r--legacy/api/router/__init__.py (renamed from qtip/drivers/__init__.py)0
-rw-r--r--legacy/api/router/mapper.py (renamed from qtip/api/router/mapper.py)2
-rw-r--r--legacy/assets/perftest/common/git_proxy_pbook.yaml (renamed from benchmarks/perftest/common/git_proxy_pbook.yaml)0
-rw-r--r--legacy/assets/perftest/common/sys_info_pbook.yaml (renamed from benchmarks/perftest/common/sys_info_pbook.yaml)0
-rw-r--r--legacy/assets/perftest/common/sys_proxy_pbook.yaml (renamed from benchmarks/perftest/common/sys_proxy_pbook.yaml)0
-rw-r--r--legacy/assets/perftest/dhrystone.yaml (renamed from benchmarks/perftest/dhrystone.yaml)0
-rw-r--r--legacy/assets/perftest/dpi.yaml (renamed from benchmarks/perftest/dpi.yaml)0
-rw-r--r--legacy/assets/perftest/etc/dpi_average.sh (renamed from benchmarks/perftest/etc/dpi_average.sh)0
-rw-r--r--legacy/assets/perftest/etc/fio_test_job (renamed from benchmarks/perftest/etc/fio_test_job)0
-rw-r--r--legacy/assets/perftest/etc/info_collect.py (renamed from benchmarks/perftest/etc/info_collect.py)0
-rw-r--r--legacy/assets/perftest/fio.yaml (renamed from benchmarks/perftest/fio.yaml)0
-rw-r--r--legacy/assets/perftest/iperf.yaml (renamed from benchmarks/perftest/iperf.yaml)0
-rw-r--r--legacy/assets/perftest/ramspeed.yaml (renamed from benchmarks/perftest/ramspeed.yaml)0
-rw-r--r--legacy/assets/perftest/ssl.yaml (renamed from benchmarks/perftest/ssl.yaml)0
-rw-r--r--legacy/assets/perftest/summary (renamed from benchmarks/perftest/summary)0
-rw-r--r--legacy/assets/perftest/whetstone.yaml (renamed from benchmarks/perftest/whetstone.yaml)0
-rw-r--r--legacy/assets/suite/compute (renamed from benchmarks/suite/compute)0
-rw-r--r--legacy/assets/suite/compute.yaml (renamed from benchmarks/suite/compute.yaml)12
-rw-r--r--legacy/assets/suite/network (renamed from benchmarks/suite/network)0
-rw-r--r--legacy/assets/suite/storage (renamed from benchmarks/suite/storage)0
-rw-r--r--legacy/assets/testplan/default/compute/dhrystone_bm.yaml (renamed from benchmarks/testplan/default/compute/dhrystone_bm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/dhrystone_vm.yaml (renamed from benchmarks/testplan/default/compute/dhrystone_vm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/dpi_bm.yaml (renamed from benchmarks/testplan/default/compute/dpi_bm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/dpi_vm.yaml (renamed from benchmarks/testplan/default/compute/dpi_vm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/ramspeed_bm.yaml (renamed from benchmarks/testplan/default/compute/ramspeed_bm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/ramspeed_vm.yaml (renamed from benchmarks/testplan/default/compute/ramspeed_vm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/ssl_bm.yaml (renamed from benchmarks/testplan/default/compute/ssl_bm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/ssl_vm.yaml (renamed from benchmarks/testplan/default/compute/ssl_vm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/whetstone_bm.yaml (renamed from benchmarks/testplan/default/compute/whetstone_bm.yaml)0
-rw-r--r--legacy/assets/testplan/default/compute/whetstone_vm.yaml (renamed from benchmarks/testplan/default/compute/whetstone_vm.yaml)0
-rw-r--r--legacy/assets/testplan/default/network/iperf_bm.yaml (renamed from benchmarks/testplan/default/network/iperf_bm.yaml)0
-rw-r--r--legacy/assets/testplan/default/network/iperf_vm.yaml (renamed from benchmarks/testplan/default/network/iperf_vm.yaml)0
-rw-r--r--legacy/assets/testplan/default/network/iperf_vm_2.yaml (renamed from benchmarks/testplan/default/network/iperf_vm_2.yaml)0
-rw-r--r--legacy/assets/testplan/default/storage/fio_bm.yaml (renamed from benchmarks/testplan/default/storage/fio_bm.yaml)0
-rw-r--r--legacy/assets/testplan/default/storage/fio_vm.yaml (renamed from benchmarks/testplan/default/storage/fio_vm.yaml)0
-rw-r--r--legacy/cli/helper.py (renamed from qtip/cli/helper.py)0
-rw-r--r--legacy/config/SampleHeat.yaml (renamed from config/SampleHeat.yaml)0
-rw-r--r--legacy/data/hosts (renamed from tests/data/hosts)0
-rw-r--r--legacy/data/my_key.pem27
-rw-r--r--legacy/data/output/hosts (renamed from tests/data/output/hosts)0
-rw-r--r--legacy/data/schema/test_bm_schema.yaml (renamed from tests/data/schema/test_bm_schema.yaml)0
-rw-r--r--legacy/data/schema/test_vm_schema.yaml (renamed from tests/data/schema/test_vm_schema.yaml)0
-rw-r--r--legacy/data/test.retry (renamed from tests/data/test.retry)0
-rw-r--r--legacy/data/test.yml (renamed from tests/data/test.yml)0
-rw-r--r--legacy/data/testplan/bm_ping.yaml (renamed from tests/data/testplan/bm_ping.yaml)0
-rw-r--r--legacy/data/testplan/bm_with_proxy.yaml (renamed from tests/data/testplan/bm_with_proxy.yaml)0
-rw-r--r--legacy/data/testplan/bm_without_proxy.yaml (renamed from tests/data/testplan/bm_without_proxy.yaml)0
-rw-r--r--legacy/data/testplan/vm.yaml (renamed from tests/data/testplan/vm.yaml)0
-rw-r--r--legacy/data/testplan/vm_error.yaml (renamed from tests/data/testplan/vm_error.yaml)0
-rw-r--r--legacy/docker/README.md (renamed from docker/README.md)0
-rw-r--r--legacy/docker/cleanup_qtip_image.sh (renamed from docker/cleanup_qtip_image.sh)0
-rw-r--r--legacy/docker/prepare_qtip_image.sh (renamed from docker/prepare_qtip_image.sh)0
-rwxr-xr-xlegacy/docker/push_db.sh (renamed from docker/push_db.sh)0
-rwxr-xr-xlegacy/docker/run_qtip.sh (renamed from docker/run_qtip.sh)0
-rw-r--r--legacy/run.py (renamed from qtip/run.py)0
-rw-r--r--legacy/scripts/__init__.py (renamed from qtip/spec/__init__.py)0
-rwxr-xr-xlegacy/scripts/cleanup_creds.sh (renamed from scripts/cleanup_creds.sh)0
-rwxr-xr-xlegacy/scripts/fetch_compute_ips.sh (renamed from scripts/fetch_compute_ips.sh)0
-rwxr-xr-xlegacy/scripts/get_env_info.sh (renamed from scripts/get_env_info.sh)0
-rwxr-xr-xlegacy/scripts/qtip_creds.sh (renamed from scripts/qtip_creds.sh)0
-rw-r--r--legacy/scripts/ref_results/__init__.py (renamed from qtip/utils/__init__.py)0
-rw-r--r--legacy/scripts/ref_results/compute_benchmarks_indices.py (renamed from scripts/ref_results/compute_benchmarks_indices.py)0
-rw-r--r--legacy/scripts/ref_results/index_calculation.py (renamed from scripts/ref_results/index_calculation.py)0
-rw-r--r--legacy/scripts/ref_results/network_benchmarks_indices.py (renamed from scripts/ref_results/network_benchmarks_indices.py)0
-rw-r--r--legacy/scripts/ref_results/reference.json (renamed from scripts/ref_results/reference.json)0
-rw-r--r--legacy/scripts/ref_results/result_accum.py (renamed from scripts/ref_results/result_accum.py)0
-rw-r--r--legacy/scripts/ref_results/storage_benchmarks_indices.py (renamed from scripts/ref_results/storage_benchmarks_indices.py)0
-rw-r--r--legacy/scripts/ref_results/suite_result.py (renamed from scripts/ref_results/suite_result.py)0
-rw-r--r--legacy/scripts/ssh_exch.exp (renamed from scripts/ssh_exch.exp)0
-rw-r--r--legacy/tests/__init__.py (renamed from qtip/utils/dashboard/__init__.py)0
-rw-r--r--legacy/tests/ansible_api_test.py (renamed from tests/unit/utils/ansible_api_test.py)0
-rw-r--r--legacy/tests/api/__init__.py (renamed from qtip/utils/report/__init__.py)0
-rw-r--r--legacy/tests/api/test_server.py (renamed from tests/unit/api/test_server.py)0
-rw-r--r--legacy/tests/args_handler_test.py (renamed from tests/unit/utils/args_handler_test.py)0
-rw-r--r--legacy/tests/cli_test.py (renamed from tests/unit/utils/cli_test.py)0
-rw-r--r--legacy/tests/create_zones_test.py (renamed from tests/unit/utils/create_zones_test.py)0
-rw-r--r--legacy/tests/driver_test.py (renamed from tests/unit/utils/driver_test.py)0
-rw-r--r--legacy/tests/env_setup_test.py (renamed from tests/unit/utils/env_setup_test.py)0
-rw-r--r--legacy/tests/functional/__init__.py (renamed from qtip/utils/transform/__init__.py)0
-rw-r--r--legacy/tests/functional/yaml_schema_test.py (renamed from tests/functional/yaml_schema_test.py)0
-rw-r--r--legacy/tests/helper/perftest.yaml (renamed from tests/data/helper/perftest.yaml)0
-rw-r--r--legacy/tests/helper/suite.yaml (renamed from tests/data/helper/suite.yaml)0
-rw-r--r--legacy/tests/helper/version.yaml (renamed from tests/data/helper/version.yaml)0
-rw-r--r--legacy/tests/spawn_vm_test.py (renamed from tests/unit/utils/spawn_vm_test.py)0
-rw-r--r--legacy/utils/__init__.py (renamed from scripts/__init__.py)0
-rw-r--r--legacy/utils/ansible_api.py (renamed from qtip/utils/ansible_api.py)0
-rw-r--r--legacy/utils/args_handler.py (renamed from qtip/utils/args_handler.py)0
-rw-r--r--legacy/utils/cli.py (renamed from qtip/utils/cli.py)0
-rw-r--r--legacy/utils/create_zones.py (renamed from qtip/utils/create_zones.py)0
-rw-r--r--legacy/utils/dashboard/__init__.py (renamed from scripts/ref_results/__init__.py)0
-rw-r--r--legacy/utils/dashboard/pushtoDB.py (renamed from qtip/utils/dashboard/pushtoDB.py)0
-rw-r--r--legacy/utils/driver.py (renamed from qtip/utils/driver.py)0
-rw-r--r--legacy/utils/env_setup.py (renamed from qtip/utils/env_setup.py)0
-rw-r--r--legacy/utils/report/__init__.py (renamed from tests/functional/__init__.py)0
-rw-r--r--legacy/utils/report/get_indices.py (renamed from qtip/utils/report/get_indices.py)0
-rw-r--r--legacy/utils/report/get_results.py (renamed from qtip/utils/report/get_results.py)0
-rw-r--r--legacy/utils/report/qtip_graph.py (renamed from qtip/utils/report/qtip_graph.py)0
-rw-r--r--legacy/utils/report/qtip_report.py (renamed from qtip/utils/report/qtip_report.py)0
-rw-r--r--legacy/utils/spawn_vm.py (renamed from qtip/utils/spawn_vm.py)0
-rw-r--r--legacy/utils/transform/__init__.py (renamed from tests/unit/api/__init__.py)0
-rw-r--r--legacy/utils/transform/dpi_transform.py (renamed from qtip/utils/transform/dpi_transform.py)0
-rw-r--r--legacy/utils/transform/final_report.py (renamed from qtip/utils/transform/final_report.py)0
-rwxr-xr-xlegacy/utils/transform/fio_transform.py (renamed from qtip/utils/transform/fio_transform.py)0
-rw-r--r--legacy/utils/transform/iperf_transform.py (renamed from qtip/utils/transform/iperf_transform.py)0
-rw-r--r--legacy/utils/transform/ramspeed_transform.py (renamed from qtip/utils/transform/ramspeed_transform.py)0
-rw-r--r--legacy/utils/transform/ssl_transform.py (renamed from qtip/utils/transform/ssl_transform.py)0
-rw-r--r--legacy/utils/transform/ubench_transform.py (renamed from qtip/utils/transform/ubench_transform.py)0
-rw-r--r--opt/__init__.py (renamed from tests/unit/utils/__init__.py)0
-rw-r--r--opt/infra/ansible.cfg (renamed from opt/servers/ansible.cfg)0
-rw-r--r--opt/infra/inventory (renamed from opt/servers/inventory)0
-rw-r--r--opt/infra/roles/docker/handlers/main.yml (renamed from opt/servers/roles/docker/handlers/main.yml)0
-rw-r--r--opt/infra/roles/docker/tasks/main.yml (renamed from opt/servers/roles/docker/tasks/main.yml)0
-rw-r--r--opt/infra/roles/elk/tasks/main.yml (renamed from opt/servers/roles/elk/tasks/main.yml)0
-rw-r--r--opt/infra/roles/mongo/tasks/main.yml (renamed from opt/servers/roles/mongo/tasks/main.yml)0
-rw-r--r--opt/infra/roles/nginx/defaults/main.yml (renamed from opt/servers/roles/nginx/defaults/main.yml)0
-rw-r--r--opt/infra/roles/nginx/handlers/main.yml (renamed from opt/servers/roles/nginx/handlers/main.yml)0
-rw-r--r--opt/infra/roles/nginx/tasks/main.yml (renamed from opt/servers/roles/nginx/tasks/main.yml)0
-rw-r--r--opt/infra/roles/nginx/templates/elk.conf.j2 (renamed from opt/servers/roles/nginx/templates/elk.conf.j2)0
-rw-r--r--opt/infra/roles/nginx/templates/qtip.conf.j2 (renamed from opt/servers/roles/nginx/templates/qtip.conf.j2)0
-rw-r--r--opt/infra/roles/nginx/templates/testapi.conf.j2 (renamed from opt/servers/roles/nginx/templates/testapi.conf.j2)0
-rw-r--r--opt/infra/roles/qtip/files/run_qtip_server.sh (renamed from opt/servers/roles/qtip/files/run_qtip_server.sh)0
-rw-r--r--opt/infra/roles/qtip/tasks/main.yml (renamed from opt/servers/roles/qtip/tasks/main.yml)0
-rw-r--r--opt/infra/roles/testapi/files/run_testapi.sh (renamed from opt/servers/roles/testapi/files/run_testapi.sh)0
-rw-r--r--opt/infra/roles/testapi/tasks/main.yml (renamed from opt/servers/roles/testapi/tasks/main.yml)0
-rw-r--r--opt/infra/roles/user/defaults/main.yml (renamed from opt/servers/roles/user/defaults/main.yml)0
-rw-r--r--opt/infra/roles/user/files/akhil.authorized_keys (renamed from opt/servers/roles/user/files/akhil.authorized_keys)0
-rw-r--r--opt/infra/roles/user/files/serena.authorized_keys (renamed from opt/servers/roles/user/files/serena.authorized_keys)0
-rw-r--r--opt/infra/roles/user/files/sudoers.d-qtip (renamed from opt/servers/roles/user/files/sudoers.d-qtip)0
-rw-r--r--opt/infra/roles/user/files/taseer.authorized_keys (renamed from opt/servers/roles/user/files/taseer.authorized_keys)0
-rw-r--r--opt/infra/roles/user/files/yujunz.authorized_keys (renamed from opt/servers/roles/user/files/yujunz.authorized_keys)0
-rw-r--r--opt/infra/roles/user/files/zhifeng.authorized_keys (renamed from opt/servers/roles/user/files/zhifeng.authorized_keys)0
-rw-r--r--opt/infra/roles/user/tasks/main.yml (renamed from opt/servers/roles/user/tasks/main.yml)0
-rw-r--r--opt/infra/test.yml (renamed from opt/servers/test.yml)0
-rw-r--r--opt/plugin/__init__.py0
-rw-r--r--opt/plugin/doctor/__init__.py0
-rw-r--r--opt/plugin/doctor/collector.yaml24
-rw-r--r--opt/plugin/doctor/sample/__init__.py0
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log4
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log11
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log2
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log6
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log3
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log421
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log9
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log3
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log1
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log1
-rwxr-xr-xopt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log1
-rw-r--r--qtip/api/cmd/server.py3
-rw-r--r--qtip/base/benchmark.py95
-rw-r--r--qtip/base/constant.py50
-rw-r--r--qtip/base/error.py36
-rw-r--r--qtip/cli/commands/cmd_metric.py (renamed from qtip/runner/case.py)28
-rw-r--r--qtip/cli/commands/cmd_perftest.py58
-rw-r--r--qtip/cli/commands/cmd_plan.py3
-rw-r--r--qtip/collector/__init__.py0
-rw-r--r--qtip/collector/base.py (renamed from qtip/drivers/base.py)4
-rw-r--r--qtip/collector/logfile.py (renamed from qtip/spec/qpi.py)10
-rw-r--r--qtip/collector/transformer/__init__.py0
-rw-r--r--qtip/collector/transformer/base.py (renamed from qtip/agent/collector.py)5
-rw-r--r--qtip/collector/transformer/timeline.py14
-rw-r--r--qtip/driver/__init__.py0
-rw-r--r--qtip/driver/ansible.py (renamed from qtip/drivers/ansible.py)0
-rw-r--r--qtip/driver/base.py20
-rw-r--r--qtip/driver/sample.py15
-rw-r--r--qtip/driver/yardstick.py (renamed from qtip/drivers/yardstick.py)0
-rw-r--r--qtip/loader/__init__.py0
-rw-r--r--qtip/loader/base.py62
-rw-r--r--qtip/loader/metric.py (renamed from qtip/spec/metric.py)6
-rw-r--r--qtip/loader/plan.py (renamed from qtip/runner/plan.py)21
-rw-r--r--qtip/loader/qpi.py38
-rw-r--r--qtip/reporter/__init__.py0
-rw-r--r--qtip/reporter/base.py14
-rw-r--r--qtip/reporter/console/__init__.py (renamed from qtip/agent/reporter.py)8
-rw-r--r--qtip/reporter/console/timeline.j28
-rw-r--r--qtip/runner/__init__.py43
-rw-r--r--qtip/runner/suite.py22
-rw-r--r--qtip/util/__init__.py0
-rw-r--r--qtip/util/formula.py29
-rw-r--r--qtip/util/logger.py (renamed from qtip/utils/logger_utils.py)0
-rw-r--r--requirements.txt1
-rw-r--r--setup.cfg5
-rw-r--r--tests/conftest.py8
-rw-r--r--tests/data/benchmarks/QPI/compute.yaml37
-rw-r--r--tests/data/benchmarks/QPI/fake-qpi.yaml9
-rw-r--r--tests/data/benchmarks/metric/dhrystone.yaml9
-rw-r--r--tests/data/benchmarks/metric/dpi.yaml (renamed from tests/data/benchmarks/metrics/dpi.yaml)0
-rw-r--r--tests/data/benchmarks/metric/fake-metric.yaml7
-rw-r--r--tests/data/benchmarks/metric/ramspeed.yaml (renamed from tests/data/benchmarks/metrics/ramspeed.yaml)0
-rw-r--r--tests/data/benchmarks/metric/ssl.yaml (renamed from tests/data/benchmarks/metrics/ssl.yaml)0
-rw-r--r--tests/data/benchmarks/metric/whetstone.yaml10
-rw-r--r--tests/data/benchmarks/metrics/dhrystone.yaml7
-rw-r--r--tests/data/benchmarks/metrics/whetstone.yaml5
-rw-r--r--tests/data/benchmarks/plan/fake-plan.yaml10
-rw-r--r--tests/data/benchmarks/plans/verification.yaml21
-rw-r--r--tests/unit/cli/test_metric.py (renamed from tests/unit/runner/case_test.py)21
-rw-r--r--tests/unit/loader/metric_test.py24
-rw-r--r--tests/unit/loader/plan_test.py (renamed from tests/unit/runner/plan_test.py)25
-rw-r--r--tests/unit/loader/qpi_test.py28
-rw-r--r--tests/unit/runner/conftest.py30
-rw-r--r--tests/unit/runner/suite_test.py27
-rw-r--r--tox.ini4
217 files changed, 1275 insertions, 410 deletions
diff --git a/.gitignore b/.gitignore
index aab79ed0..7c7892db 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,3 +66,10 @@ target/
/docs_build/
/docs_output/
.idea
+
+# setup.py cache
+AUTHORS
+ChangeLog
+
+# unignore sample logfiles
+!/opt/plugin/doctor/sample/*/*
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 93c8bc9d..a4929172 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -13,19 +13,31 @@ Peer review is the most important communication channel between developers.
Every subtle change to the code or document **MUST** be reviewed before
submission.
-Add group `qtip-reviewers` in [gerrit][gr] when you consider a patch set is ready.
+Add group `qtip-reviewers` in [gerrit][gr] when you consider a patch set is
+ready.
-Please make sure there is at least one `+1` or `+2` from others before
-submitting a patch set.
-Note: only members in `ldap/opnfv-gerrit-qtip-submitters` have permission
-to submit. The current members are listed in [INFO][if].
+Rule for Submit
+---------------
+
+Only members in `ldap/opnfv-gerrit-qtip-submitters`, i.e. committers have
+permission to submit. The current committers are listed in [INFO][if].
+
+Before submitting a patch set, please make sure one of the following criteria is
+met:
+
+* **Normal**: at least one `+2` from others
+* **Fast Track**: no objection from other committers for one or more working day
+
+NOTE: we introduce **fast track** because currently there are not enough active
+committers reviewing the patch sets.
Active Reviewers
----------------
Current list of active reviewers in gerrit group `qtip-reviewers`
+* Akhil Batra <akhil.batra@research.iiit.ac.in>
* Serena Feng <feng.xiaowei@zte.com.cn>
* Taseer Ahmed <taseer94@gmail.com>
* Yujun Zhang <zhang.yujunz@zte.com.cn>
diff --git a/DEVELOP.md b/DEVELOP.md
new file mode 100644
index 00000000..9ec3a93d
--- /dev/null
+++ b/DEVELOP.md
@@ -0,0 +1,101 @@
+# QTIP Developer Guide
+
+This guide is about how to **develop** QTIP.
+
+If you just want to use it for performance benchmark, check the user guide
+instead.
+
+## Getting Started
+
+### Source code
+
+```bash
+~$ git clone https://gerrit.opnfv.org/gerrit/qtip
+```
+
+### VirtualEnv
+
+It is recommended to use [virtualenv](https://virtualenv.pypa.io) to isolate
+your development environment from system, especially when you are working on
+several different projects.
+
+### Testing
+
+QTIP use [tox](https://tox.readthedocs.io) to automate the testing tasks
+
+```bash
+$ pip install tox
+$ tox
+```
+
+## Architecture
+
+**TODO**: move to design spec
+
+QTIP has a flexible architecture to allow different deployment mode
+
+- **Standalone**: full feature performance benchmark platform.
+- **Agent**: minimal agent driven by external test runners.
+
+### Standalone Mode (Solo)
+
+QTIP instance deployed in container, VM or host generate benchmark report and
+push data to Indices Hub for storage and visualization.
+
+![solo](https://wiki.opnfv.org/download/attachments/8687017/Standalone.png?api=v2)
+
+### Agent Mode (Melody)
+
+QTIP Collector and Reporter driven by external test framework or runner such as
+[yardstick](https://wiki.opnfv.org/display/yardstick),
+[pytest](http://doc.pytest.org/) and etc.
+
+![melody](https://wiki.opnfv.org/download/attachments/8687017/Agent.png?api=v2)
+
+## Core Modules
+
+TBD
+
+- loader
+- runner
+- collector
+- reporter
+
+## Drivers
+
+TBD
+
+- ansible
+- yardstick
+
+## Interfaces
+
+### Agent
+
+TBD
+
+### CLI
+
+TBD
+
+### API
+
+TBD
+
+## Assets
+
+**TODO**: move to user guide
+
+- benchmark plan
+- QPI spec
+- metric spec
+
+## Docker Image
+
+TBD
+
+## Annex
+
+### Directories
+
+TBD
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 369fa6c8..f2a543e6 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,8 +2,7 @@
#####Docker container for QTIP############
##########################################
-
-FROM ubuntu:14.04
+FROM ubuntu:16.04
MAINTAINER Yujun Zhang <zhang.yujunz@zte.com.cn>
LABEL version="0.1" description="OPNFV QTIP Docker container"
@@ -17,31 +16,31 @@ WORKDIR /home/opnfv
# Packaged Dependencies
RUN apt-get update && apt-get install -y \
-software-properties-common \
-wget \
-git \
-gcc \
-ssh \
-expect \
-python-matplotlib \
-python-dev \
-python-pip \
-libjpeg62 \
-libjpeg62-dev \
-zlib1g-dev \
-python-tk \
-curl \
-supervisor \
-python-setuptools \
---no-install-recommends
-
+ software-properties-common \
+ wget \
+ git \
+ gcc \
+ ssh \
+ expect \
+ python-matplotlib \
+ python-dev \
+ python-pip \
+ libjpeg62 \
+ libjpeg62-dev \
+ zlib1g-dev \
+ python-tk \
+ curl \
+ supervisor \
+ python-setuptools \
+ --no-install-recommends \
+&& rm -rf /var/lib/apt/lists/*
+
+RUN pip install -U pip
RUN pip install 'setuptools>=17.1'
RUN apt-add-repository ppa:ansible/ansible -y
RUN apt-key update -y
-RUN apt-get update -y
-RUN apt-get install ansible --force-yes -y
-
+RUN apt-get update && apt-get install ansible -y
RUN mkdir -p ${REPOS_DIR}
RUN mkdir -p /root/.ssh
@@ -54,7 +53,6 @@ RUN chmod 700 /root/.ssh
COPY ansible.cfg.default /root/.ansible.cfg
#Cloning Repos
-
RUN git config --global http.sslVerify false
RUN git clone -b $BRANCH https://gerrit.opnfv.org/gerrit/qtip $REPOS_DIR/qtip
RUN git clone https://gerrit.opnfv.org/gerrit/releng $REPOS_DIR/releng
@@ -62,7 +60,6 @@ RUN git clone https://gerrit.opnfv.org/gerrit/releng $REPOS_DIR/releng
RUN pip install -r $REPOS_DIR/qtip/requirements.txt
#Config supervisor
-
RUN mkdir -p /var/log/supervisor
RUN locale-gen en_US en_US.UTF-8
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
diff --git a/docs/designspec/index.rst b/docs/designspec/index.rst
index d3bed217..b6dd0c01 100644
--- a/docs/designspec/index.rst
+++ b/docs/designspec/index.rst
@@ -12,3 +12,4 @@ QTIP Design Specifications
dashboard.rst
compute_QPI.rst
+ integration_with_yardstick.rst
diff --git a/docs/designspec/integration_with_yardstick.rst b/docs/designspec/integration_with_yardstick.rst
new file mode 100644
index 00000000..a8298d6f
--- /dev/null
+++ b/docs/designspec/integration_with_yardstick.rst
@@ -0,0 +1,92 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) 2016 ZTE Corp.
+
+
+**************************
+Integration with Yardstick
+**************************
+
+Problem description
+===================
+
+For each specified QPI [1]_, QTIP needs to select a suite of test cases and collect
+required test results. Based on these results, Qtip calculates the score.
+
+Proposed change
+===============
+Qtip has a flexible architecture [2]_ to support different mode: standalone and agent.
+It is recommended to use **agent mode** to work with existing test runners. Yardstick will
+act as a runner to generate test result and trigger Qtip agent on the completion of test.
+
+
+Work Items in Yardstick
+-----------------------
+
+1. Create a customized suite in Yardstick
+
+Yardstick not only has many existing suites but also support customized suites. Qtip could
+create a suite named **Qtip-PoC** in Yardstick repo to verify workflow of Qtip agent mode.
+
+2. Launch Qtip in Yardstick
+
+Whether to launch Qtip will be determined by checking the existence of OS environment
+variable *QTIP*. If it exists, Qtip will be launched by using Yardstick CLI
+`yardstick plugin install` [3]_.
+
+3. Yardstick interacts with Qtip
+
+See
+`Yardstick-Qtip+integration <https://wiki.opnfv.org/display/yardstick/Yardstick-Qtip+integration>`_
+for details.
+
+Work Items in Qtip
+------------------
+
+1. Provide an API for Yardstick to post test result and environment info
+
+After completing test execution, Yardstick will post test result and enviroment info with
+JSON format via QTIP API. See
+`Yardstick-Qtip+integration <https://wiki.opnfv.org/display/yardstick/Yardstick-Qtip+integration>`_
+for details.
+
+2. Parse yardstick test result
+
+When Qtip agent receive Yarstick test result and enviroment info, Qtip agent will extract
+metrics which is definded in metric spec configuration file. Based on these metrics, Qtip
+agent will caculate QPI.
+
+3. Provide an API for querying QPI
+
+Qtip will provide an API for querying QPI. See
+`Yardstick-Qtip+integration <https://wiki.opnfv.org/display/yardstick/Yardstick-Qtip+integration>`_
+for details.
+
+Implementation
+==============
+
+Assignee(s)
+-----------
+
+*Primary assignee:*
+ wu.zhihui
+
+*Other contributors*
+ TBD
+
+Testing
+=======
+
+The changes will be covered by new unit test.
+
+Documentation
+=============
+
+TBD
+
+Reference
+=========
+
+.. [1] QTIP performance index
+.. [2] https://wiki.opnfv.org/display/qtip/Architecture
+.. [3] https://wiki.opnfv.org/display/yardstick/How+to+install+a+plug-in+into+Yardstick
diff --git a/legacy/DO-NOT-DELETE b/legacy/DO-NOT-DELETE
new file mode 100644
index 00000000..fdecaad1
--- /dev/null
+++ b/legacy/DO-NOT-DELETE
@@ -0,0 +1,2 @@
+The legacy code is no longer maintained. But they should be kept until we finish
+migration to new architecture.
diff --git a/qtip/agent/__init__.py b/legacy/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/agent/__init__.py
+++ b/legacy/__init__.py
diff --git a/qtip/api/handler/__init__.py b/legacy/api/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/api/handler/__init__.py
+++ b/legacy/api/__init__.py
diff --git a/qtip/api/model/__init__.py b/legacy/api/handler/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/api/model/__init__.py
+++ b/legacy/api/handler/__init__.py
diff --git a/qtip/api/handler/db.py b/legacy/api/handler/db.py
index 24fc27a5..24fc27a5 100644
--- a/qtip/api/handler/db.py
+++ b/legacy/api/handler/db.py
diff --git a/qtip/api/handler/job_handler.py b/legacy/api/handler/job_handler.py
index eade9df7..b75da5ff 100644
--- a/qtip/api/handler/job_handler.py
+++ b/legacy/api/handler/job_handler.py
@@ -3,11 +3,11 @@ from copy import copy
from flask_restful import Resource, reqparse
from flask_restful_swagger import swagger
-from werkzeug.exceptions import abort
-
-from qtip.api.handler import db, result_handler
from qtip.api.model.job_model import JobResponseModel
from qtip.utils import args_handler as args_handler
+from werkzeug.exceptions import abort
+
+from legacy.api.handler import db, result_handler
class Job(Resource):
diff --git a/qtip/api/handler/result_handler.py b/legacy/api/handler/result_handler.py
index 3d1d592e..3d1d592e 100644
--- a/qtip/api/handler/result_handler.py
+++ b/legacy/api/handler/result_handler.py
diff --git a/qtip/api/router/__init__.py b/legacy/api/model/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/api/router/__init__.py
+++ b/legacy/api/model/__init__.py
diff --git a/qtip/api/model/job_model.py b/legacy/api/model/job_model.py
index f7eb9fda..f7eb9fda 100644
--- a/qtip/api/model/job_model.py
+++ b/legacy/api/model/job_model.py
diff --git a/qtip/drivers/__init__.py b/legacy/api/router/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/drivers/__init__.py
+++ b/legacy/api/router/__init__.py
diff --git a/qtip/api/router/mapper.py b/legacy/api/router/mapper.py
index a5f029ac..8839f3f5 100644
--- a/qtip/api/router/mapper.py
+++ b/legacy/api/router/mapper.py
@@ -1,4 +1,4 @@
-from qtip.api.handler.job_handler import Job, JobList
+from legacy.api.handler import Job, JobList
mappers = [
diff --git a/benchmarks/perftest/common/git_proxy_pbook.yaml b/legacy/assets/perftest/common/git_proxy_pbook.yaml
index 5cb6f450..5cb6f450 100644
--- a/benchmarks/perftest/common/git_proxy_pbook.yaml
+++ b/legacy/assets/perftest/common/git_proxy_pbook.yaml
diff --git a/benchmarks/perftest/common/sys_info_pbook.yaml b/legacy/assets/perftest/common/sys_info_pbook.yaml
index 5c2d8f79..5c2d8f79 100644
--- a/benchmarks/perftest/common/sys_info_pbook.yaml
+++ b/legacy/assets/perftest/common/sys_info_pbook.yaml
diff --git a/benchmarks/perftest/common/sys_proxy_pbook.yaml b/legacy/assets/perftest/common/sys_proxy_pbook.yaml
index bf4a8ccb..bf4a8ccb 100644
--- a/benchmarks/perftest/common/sys_proxy_pbook.yaml
+++ b/legacy/assets/perftest/common/sys_proxy_pbook.yaml
diff --git a/benchmarks/perftest/dhrystone.yaml b/legacy/assets/perftest/dhrystone.yaml
index 896aadfc..896aadfc 100644
--- a/benchmarks/perftest/dhrystone.yaml
+++ b/legacy/assets/perftest/dhrystone.yaml
diff --git a/benchmarks/perftest/dpi.yaml b/legacy/assets/perftest/dpi.yaml
index 2a10bef6..2a10bef6 100644
--- a/benchmarks/perftest/dpi.yaml
+++ b/legacy/assets/perftest/dpi.yaml
diff --git a/benchmarks/perftest/etc/dpi_average.sh b/legacy/assets/perftest/etc/dpi_average.sh
index 405d3ff6..405d3ff6 100644
--- a/benchmarks/perftest/etc/dpi_average.sh
+++ b/legacy/assets/perftest/etc/dpi_average.sh
diff --git a/benchmarks/perftest/etc/fio_test_job b/legacy/assets/perftest/etc/fio_test_job
index 6817abca..6817abca 100644
--- a/benchmarks/perftest/etc/fio_test_job
+++ b/legacy/assets/perftest/etc/fio_test_job
diff --git a/benchmarks/perftest/etc/info_collect.py b/legacy/assets/perftest/etc/info_collect.py
index 3fc35d5a..3fc35d5a 100644
--- a/benchmarks/perftest/etc/info_collect.py
+++ b/legacy/assets/perftest/etc/info_collect.py
diff --git a/benchmarks/perftest/fio.yaml b/legacy/assets/perftest/fio.yaml
index 0da9407d..0da9407d 100644
--- a/benchmarks/perftest/fio.yaml
+++ b/legacy/assets/perftest/fio.yaml
diff --git a/benchmarks/perftest/iperf.yaml b/legacy/assets/perftest/iperf.yaml
index a1755f7c..a1755f7c 100644
--- a/benchmarks/perftest/iperf.yaml
+++ b/legacy/assets/perftest/iperf.yaml
diff --git a/benchmarks/perftest/ramspeed.yaml b/legacy/assets/perftest/ramspeed.yaml
index 83ecd8bf..83ecd8bf 100644
--- a/benchmarks/perftest/ramspeed.yaml
+++ b/legacy/assets/perftest/ramspeed.yaml
diff --git a/benchmarks/perftest/ssl.yaml b/legacy/assets/perftest/ssl.yaml
index 6002bfff..6002bfff 100644
--- a/benchmarks/perftest/ssl.yaml
+++ b/legacy/assets/perftest/ssl.yaml
diff --git a/benchmarks/perftest/summary b/legacy/assets/perftest/summary
index 5891408c..5891408c 100644
--- a/benchmarks/perftest/summary
+++ b/legacy/assets/perftest/summary
diff --git a/benchmarks/perftest/whetstone.yaml b/legacy/assets/perftest/whetstone.yaml
index d6eae85f..d6eae85f 100644
--- a/benchmarks/perftest/whetstone.yaml
+++ b/legacy/assets/perftest/whetstone.yaml
diff --git a/benchmarks/suite/compute b/legacy/assets/suite/compute
index 3bf1b184..3bf1b184 100644
--- a/benchmarks/suite/compute
+++ b/legacy/assets/suite/compute
diff --git a/benchmarks/suite/compute.yaml b/legacy/assets/suite/compute.yaml
index 0761a87b..197d5720 100644
--- a/benchmarks/suite/compute.yaml
+++ b/legacy/assets/suite/compute.yaml
@@ -1,12 +1,12 @@
QPI: compute
description: sample performance index of computing
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
section:
- name: Integer
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: dhrystone
workloads:
@@ -14,7 +14,7 @@ section:
- multi_cpu
- name: Floating
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: whetstone
workloads:
@@ -22,7 +22,7 @@ section:
- multi_cpu
- name: Memory
weight: 0.2
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: ramspeed
workloads:
@@ -30,7 +30,7 @@ section:
- float: [add, average, copy, scale, triad]
- name: DPI
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: dpi
workloads:
@@ -38,7 +38,7 @@ section:
- pps
- name: SSL
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: ssl
workloads:
diff --git a/benchmarks/suite/network b/legacy/assets/suite/network
index 58ce5cb9..58ce5cb9 100644
--- a/benchmarks/suite/network
+++ b/legacy/assets/suite/network
diff --git a/benchmarks/suite/storage b/legacy/assets/suite/storage
index f3068dd5..f3068dd5 100644
--- a/benchmarks/suite/storage
+++ b/legacy/assets/suite/storage
diff --git a/benchmarks/testplan/default/compute/dhrystone_bm.yaml b/legacy/assets/testplan/default/compute/dhrystone_bm.yaml
index 64741537..64741537 100644
--- a/benchmarks/testplan/default/compute/dhrystone_bm.yaml
+++ b/legacy/assets/testplan/default/compute/dhrystone_bm.yaml
diff --git a/benchmarks/testplan/default/compute/dhrystone_vm.yaml b/legacy/assets/testplan/default/compute/dhrystone_vm.yaml
index 8f5a4165..8f5a4165 100644
--- a/benchmarks/testplan/default/compute/dhrystone_vm.yaml
+++ b/legacy/assets/testplan/default/compute/dhrystone_vm.yaml
diff --git a/benchmarks/testplan/default/compute/dpi_bm.yaml b/legacy/assets/testplan/default/compute/dpi_bm.yaml
index 14cb6d0d..14cb6d0d 100644
--- a/benchmarks/testplan/default/compute/dpi_bm.yaml
+++ b/legacy/assets/testplan/default/compute/dpi_bm.yaml
diff --git a/benchmarks/testplan/default/compute/dpi_vm.yaml b/legacy/assets/testplan/default/compute/dpi_vm.yaml
index 10e86993..10e86993 100644
--- a/benchmarks/testplan/default/compute/dpi_vm.yaml
+++ b/legacy/assets/testplan/default/compute/dpi_vm.yaml
diff --git a/benchmarks/testplan/default/compute/ramspeed_bm.yaml b/legacy/assets/testplan/default/compute/ramspeed_bm.yaml
index ed8fc700..ed8fc700 100644
--- a/benchmarks/testplan/default/compute/ramspeed_bm.yaml
+++ b/legacy/assets/testplan/default/compute/ramspeed_bm.yaml
diff --git a/benchmarks/testplan/default/compute/ramspeed_vm.yaml b/legacy/assets/testplan/default/compute/ramspeed_vm.yaml
index a6a4363f..a6a4363f 100644
--- a/benchmarks/testplan/default/compute/ramspeed_vm.yaml
+++ b/legacy/assets/testplan/default/compute/ramspeed_vm.yaml
diff --git a/benchmarks/testplan/default/compute/ssl_bm.yaml b/legacy/assets/testplan/default/compute/ssl_bm.yaml
index cc5a8903..cc5a8903 100644
--- a/benchmarks/testplan/default/compute/ssl_bm.yaml
+++ b/legacy/assets/testplan/default/compute/ssl_bm.yaml
diff --git a/benchmarks/testplan/default/compute/ssl_vm.yaml b/legacy/assets/testplan/default/compute/ssl_vm.yaml
index 37824896..37824896 100644
--- a/benchmarks/testplan/default/compute/ssl_vm.yaml
+++ b/legacy/assets/testplan/default/compute/ssl_vm.yaml
diff --git a/benchmarks/testplan/default/compute/whetstone_bm.yaml b/legacy/assets/testplan/default/compute/whetstone_bm.yaml
index 3c128b60..3c128b60 100644
--- a/benchmarks/testplan/default/compute/whetstone_bm.yaml
+++ b/legacy/assets/testplan/default/compute/whetstone_bm.yaml
diff --git a/benchmarks/testplan/default/compute/whetstone_vm.yaml b/legacy/assets/testplan/default/compute/whetstone_vm.yaml
index 0f1e8748..0f1e8748 100644
--- a/benchmarks/testplan/default/compute/whetstone_vm.yaml
+++ b/legacy/assets/testplan/default/compute/whetstone_vm.yaml
diff --git a/benchmarks/testplan/default/network/iperf_bm.yaml b/legacy/assets/testplan/default/network/iperf_bm.yaml
index 3aa8310d..3aa8310d 100644
--- a/benchmarks/testplan/default/network/iperf_bm.yaml
+++ b/legacy/assets/testplan/default/network/iperf_bm.yaml
diff --git a/benchmarks/testplan/default/network/iperf_vm.yaml b/legacy/assets/testplan/default/network/iperf_vm.yaml
index 49bf13ad..49bf13ad 100644
--- a/benchmarks/testplan/default/network/iperf_vm.yaml
+++ b/legacy/assets/testplan/default/network/iperf_vm.yaml
diff --git a/benchmarks/testplan/default/network/iperf_vm_2.yaml b/legacy/assets/testplan/default/network/iperf_vm_2.yaml
index c5b94715..c5b94715 100644
--- a/benchmarks/testplan/default/network/iperf_vm_2.yaml
+++ b/legacy/assets/testplan/default/network/iperf_vm_2.yaml
diff --git a/benchmarks/testplan/default/storage/fio_bm.yaml b/legacy/assets/testplan/default/storage/fio_bm.yaml
index d0001ea2..d0001ea2 100644
--- a/benchmarks/testplan/default/storage/fio_bm.yaml
+++ b/legacy/assets/testplan/default/storage/fio_bm.yaml
diff --git a/benchmarks/testplan/default/storage/fio_vm.yaml b/legacy/assets/testplan/default/storage/fio_vm.yaml
index 0e7f121b..0e7f121b 100644
--- a/benchmarks/testplan/default/storage/fio_vm.yaml
+++ b/legacy/assets/testplan/default/storage/fio_vm.yaml
diff --git a/qtip/cli/helper.py b/legacy/cli/helper.py
index acfecf8d..acfecf8d 100644
--- a/qtip/cli/helper.py
+++ b/legacy/cli/helper.py
diff --git a/config/SampleHeat.yaml b/legacy/config/SampleHeat.yaml
index a42cdb79..a42cdb79 100644
--- a/config/SampleHeat.yaml
+++ b/legacy/config/SampleHeat.yaml
diff --git a/tests/data/hosts b/legacy/data/hosts
index 0a0ac539..0a0ac539 100644
--- a/tests/data/hosts
+++ b/legacy/data/hosts
diff --git a/legacy/data/my_key.pem b/legacy/data/my_key.pem
new file mode 100644
index 00000000..d7c96f24
--- /dev/null
+++ b/legacy/data/my_key.pem
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAntmA9ybqcxQKr9R3iTbNr+89ZJwlt5+gLbT8VR9sUAYCEEJn
+xX7DX5djpSdQ1OoxJun/HE0ByKPXCIqGq3sHnxQ/3Wh80UGlyiSXgS8/p8NlfgPr
+DIDuVNhKJlsobsfTVXL789i512rqf2zFBWfoesFgZee6ACrSYN3hdNICFOwtbmHA
+g+xEs00yGmbcFDuBQnDeR2yPpV6G4AtrU5zwliVj7fMzrB5w9De20ydbFqxjPdOD
+gbfwrhQQs82pv7vfJCFByjsSlwP4mcznKgOt/aO2y/B1ZvL+dOsCi8D3H6Ggrg7R
+wmiIRaCijYX4SycYxdn+RkZpp9g0AyR/potcKwIDAQABAoIBADncgF2Gj1/brQjf
+G6ufiszLGFHNju+T9YSwqDlZeNqtVZMWnTYTNpdbTbCa4Zast7q1AXgNlNjA1VMH
+IobUCbKobZr4tH6Eqx82tPkZfNZfFlkQlE25qRa+skPGcLPpldFKUPxBtXACQeio
+dhvB/ay5Q6PbBKWK85bVO4qR9LApxHCDQgovGtasUF1/wu5z4vOCP3KLtlE268m+
+XJ/4IZX2erBOfxOGPqp+qDK8FTP1NcuGFEkhUa8Tr406CIXptTDyPc+bMZANIGDM
+hhww6VR5aHxsHWgu1UITVZuDh5fJ8U6n2utqTm/QLT25DUPKoTJSAjLet7fnm4Cw
+hee7GgECgYEAziiWq4peA9WrJYTlR8bNRLgsMjZx/AphyJQpFCQsgCMuB9wIluTT
+5Di4jvdvKclsFTh6foxnltltX6O8tL+vrzfbK9fBB3A2T1V1sDMOznhI47Mmoqk6
+9le6aMmpTIhNFJMu1DpsuoxRLIJaNkWLCwWaIE1ZggqCgbjIdMOJt20CgYEAxUDo
+DUf0zi/qYZVVTnziubcqu0kjUWhmeEiv08WFst8l2fAwHBILNms5ot4TY5MlFxta
+vLsWaXC7o6tqofSHF52cD8AbUWEnvK2NtCA7gW+0xBtQEPqrtItA0YjblFXw7qwb
+QUValSo535bFYJjp5foJxTZXg9rErF4iUgExqvcCgYBK5t5PNvePhxsmh5FSMb20
+oQGVwMhLHW6HlKUUJ0xvxdhcjKomQL9npOAROX8O/JqjXyDKR7L/UMH3RKM0PIzV
+KYMc+8erLy7cRh17RiG0DGnXKOj5omjExfz5Q4OaRc9TkWIrQ4rrgD2h4T5rh3rM
+J13nLJM/txfRj9Rs7/piRQKBgCm3RFwqr5c7gvIIRvZGaxyjpCHfodyBm8osdSLw
+Two5LyQcK9CD8GMd4h6ToL4aTGnKmzH2zxKViNlgUzjVIgzYhDzAgAaB6Yl5mtJR
+TsRc/3nJ3PD6Un3oRVkK/IUud7nCJDF1nWaWe47RsARx0mWUr6RJjdCQ368kaVW2
+cu6ZAoGBALD4Gw+AsH6/za7gmRIqlzBURHpHCgKutQKv3UbD1hVc5tDAoYwBjjod
+AVY8N8+AmX/nmJOVcch/dAnICkLrZn5Bm/q52/3xCAlnGkwNHyW4G99lZCfNEQrx
+sVAZ4FNrtMC9Xwtj0o73ojqKP4gxVkljbybnbuyNSXZo14EbXuJU
+-----END RSA PRIVATE KEY-----
diff --git a/tests/data/output/hosts b/legacy/data/output/hosts
index 9b47df0e..9b47df0e 100644
--- a/tests/data/output/hosts
+++ b/legacy/data/output/hosts
diff --git a/tests/data/schema/test_bm_schema.yaml b/legacy/data/schema/test_bm_schema.yaml
index a7c27e3f..a7c27e3f 100644
--- a/tests/data/schema/test_bm_schema.yaml
+++ b/legacy/data/schema/test_bm_schema.yaml
diff --git a/tests/data/schema/test_vm_schema.yaml b/legacy/data/schema/test_vm_schema.yaml
index 524f8fe4..524f8fe4 100644
--- a/tests/data/schema/test_vm_schema.yaml
+++ b/legacy/data/schema/test_vm_schema.yaml
diff --git a/tests/data/test.retry b/legacy/data/test.retry
index 7b9ad531..7b9ad531 100644
--- a/tests/data/test.retry
+++ b/legacy/data/test.retry
diff --git a/tests/data/test.yml b/legacy/data/test.yml
index 270e86fd..270e86fd 100644
--- a/tests/data/test.yml
+++ b/legacy/data/test.yml
diff --git a/tests/data/testplan/bm_ping.yaml b/legacy/data/testplan/bm_ping.yaml
index 41d696e2..41d696e2 100644
--- a/tests/data/testplan/bm_ping.yaml
+++ b/legacy/data/testplan/bm_ping.yaml
diff --git a/tests/data/testplan/bm_with_proxy.yaml b/legacy/data/testplan/bm_with_proxy.yaml
index 1d73300b..1d73300b 100644
--- a/tests/data/testplan/bm_with_proxy.yaml
+++ b/legacy/data/testplan/bm_with_proxy.yaml
diff --git a/tests/data/testplan/bm_without_proxy.yaml b/legacy/data/testplan/bm_without_proxy.yaml
index a9ae3b71..a9ae3b71 100644
--- a/tests/data/testplan/bm_without_proxy.yaml
+++ b/legacy/data/testplan/bm_without_proxy.yaml
diff --git a/tests/data/testplan/vm.yaml b/legacy/data/testplan/vm.yaml
index 4c8453ca..4c8453ca 100644
--- a/tests/data/testplan/vm.yaml
+++ b/legacy/data/testplan/vm.yaml
diff --git a/tests/data/testplan/vm_error.yaml b/legacy/data/testplan/vm_error.yaml
index f13d3a00..f13d3a00 100644
--- a/tests/data/testplan/vm_error.yaml
+++ b/legacy/data/testplan/vm_error.yaml
diff --git a/docker/README.md b/legacy/docker/README.md
index 35ac0935..35ac0935 100644
--- a/docker/README.md
+++ b/legacy/docker/README.md
diff --git a/docker/cleanup_qtip_image.sh b/legacy/docker/cleanup_qtip_image.sh
index 9c2b59db..9c2b59db 100644
--- a/docker/cleanup_qtip_image.sh
+++ b/legacy/docker/cleanup_qtip_image.sh
diff --git a/docker/prepare_qtip_image.sh b/legacy/docker/prepare_qtip_image.sh
index 4095c806..4095c806 100644
--- a/docker/prepare_qtip_image.sh
+++ b/legacy/docker/prepare_qtip_image.sh
diff --git a/docker/push_db.sh b/legacy/docker/push_db.sh
index 50341eac..50341eac 100755
--- a/docker/push_db.sh
+++ b/legacy/docker/push_db.sh
diff --git a/docker/run_qtip.sh b/legacy/docker/run_qtip.sh
index 98abf139..98abf139 100755
--- a/docker/run_qtip.sh
+++ b/legacy/docker/run_qtip.sh
diff --git a/qtip/run.py b/legacy/run.py
index a2c26eda..a2c26eda 100644
--- a/qtip/run.py
+++ b/legacy/run.py
diff --git a/qtip/spec/__init__.py b/legacy/scripts/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/spec/__init__.py
+++ b/legacy/scripts/__init__.py
diff --git a/scripts/cleanup_creds.sh b/legacy/scripts/cleanup_creds.sh
index b4eee924..b4eee924 100755
--- a/scripts/cleanup_creds.sh
+++ b/legacy/scripts/cleanup_creds.sh
diff --git a/scripts/fetch_compute_ips.sh b/legacy/scripts/fetch_compute_ips.sh
index 4bdc9a48..4bdc9a48 100755
--- a/scripts/fetch_compute_ips.sh
+++ b/legacy/scripts/fetch_compute_ips.sh
diff --git a/scripts/get_env_info.sh b/legacy/scripts/get_env_info.sh
index cd49ac87..cd49ac87 100755
--- a/scripts/get_env_info.sh
+++ b/legacy/scripts/get_env_info.sh
diff --git a/scripts/qtip_creds.sh b/legacy/scripts/qtip_creds.sh
index af051ac5..af051ac5 100755
--- a/scripts/qtip_creds.sh
+++ b/legacy/scripts/qtip_creds.sh
diff --git a/qtip/utils/__init__.py b/legacy/scripts/ref_results/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/utils/__init__.py
+++ b/legacy/scripts/ref_results/__init__.py
diff --git a/scripts/ref_results/compute_benchmarks_indices.py b/legacy/scripts/ref_results/compute_benchmarks_indices.py
index 0b6eae36..0b6eae36 100644
--- a/scripts/ref_results/compute_benchmarks_indices.py
+++ b/legacy/scripts/ref_results/compute_benchmarks_indices.py
diff --git a/scripts/ref_results/index_calculation.py b/legacy/scripts/ref_results/index_calculation.py
index 95c3c4a6..95c3c4a6 100644
--- a/scripts/ref_results/index_calculation.py
+++ b/legacy/scripts/ref_results/index_calculation.py
diff --git a/scripts/ref_results/network_benchmarks_indices.py b/legacy/scripts/ref_results/network_benchmarks_indices.py
index c19d18eb..c19d18eb 100644
--- a/scripts/ref_results/network_benchmarks_indices.py
+++ b/legacy/scripts/ref_results/network_benchmarks_indices.py
diff --git a/scripts/ref_results/reference.json b/legacy/scripts/ref_results/reference.json
index cfcbfc3b..cfcbfc3b 100644
--- a/scripts/ref_results/reference.json
+++ b/legacy/scripts/ref_results/reference.json
diff --git a/scripts/ref_results/result_accum.py b/legacy/scripts/ref_results/result_accum.py
index 6cd55886..6cd55886 100644
--- a/scripts/ref_results/result_accum.py
+++ b/legacy/scripts/ref_results/result_accum.py
diff --git a/scripts/ref_results/storage_benchmarks_indices.py b/legacy/scripts/ref_results/storage_benchmarks_indices.py
index a5aef638..a5aef638 100644
--- a/scripts/ref_results/storage_benchmarks_indices.py
+++ b/legacy/scripts/ref_results/storage_benchmarks_indices.py
diff --git a/scripts/ref_results/suite_result.py b/legacy/scripts/ref_results/suite_result.py
index 66213391..66213391 100644
--- a/scripts/ref_results/suite_result.py
+++ b/legacy/scripts/ref_results/suite_result.py
diff --git a/scripts/ssh_exch.exp b/legacy/scripts/ssh_exch.exp
index c52140b7..c52140b7 100644
--- a/scripts/ssh_exch.exp
+++ b/legacy/scripts/ssh_exch.exp
diff --git a/qtip/utils/dashboard/__init__.py b/legacy/tests/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/utils/dashboard/__init__.py
+++ b/legacy/tests/__init__.py
diff --git a/tests/unit/utils/ansible_api_test.py b/legacy/tests/ansible_api_test.py
index 6f286fc3..6f286fc3 100644
--- a/tests/unit/utils/ansible_api_test.py
+++ b/legacy/tests/ansible_api_test.py
diff --git a/qtip/utils/report/__init__.py b/legacy/tests/api/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/utils/report/__init__.py
+++ b/legacy/tests/api/__init__.py
diff --git a/tests/unit/api/test_server.py b/legacy/tests/api/test_server.py
index e9364d3d..e9364d3d 100644
--- a/tests/unit/api/test_server.py
+++ b/legacy/tests/api/test_server.py
diff --git a/tests/unit/utils/args_handler_test.py b/legacy/tests/args_handler_test.py
index dceca1f5..dceca1f5 100644
--- a/tests/unit/utils/args_handler_test.py
+++ b/legacy/tests/args_handler_test.py
diff --git a/tests/unit/utils/cli_test.py b/legacy/tests/cli_test.py
index 0f3e4158..0f3e4158 100644
--- a/tests/unit/utils/cli_test.py
+++ b/legacy/tests/cli_test.py
diff --git a/tests/unit/utils/create_zones_test.py b/legacy/tests/create_zones_test.py
index dcfff5ec..dcfff5ec 100644
--- a/tests/unit/utils/create_zones_test.py
+++ b/legacy/tests/create_zones_test.py
diff --git a/tests/unit/utils/driver_test.py b/legacy/tests/driver_test.py
index 432ce1ae..432ce1ae 100644
--- a/tests/unit/utils/driver_test.py
+++ b/legacy/tests/driver_test.py
diff --git a/tests/unit/utils/env_setup_test.py b/legacy/tests/env_setup_test.py
index dea48190..dea48190 100644
--- a/tests/unit/utils/env_setup_test.py
+++ b/legacy/tests/env_setup_test.py
diff --git a/qtip/utils/transform/__init__.py b/legacy/tests/functional/__init__.py
index e69de29b..e69de29b 100644
--- a/qtip/utils/transform/__init__.py
+++ b/legacy/tests/functional/__init__.py
diff --git a/tests/functional/yaml_schema_test.py b/legacy/tests/functional/yaml_schema_test.py
index a975dca6..a975dca6 100644
--- a/tests/functional/yaml_schema_test.py
+++ b/legacy/tests/functional/yaml_schema_test.py
diff --git a/tests/data/helper/perftest.yaml b/legacy/tests/helper/perftest.yaml
index 26c58452..26c58452 100644
--- a/tests/data/helper/perftest.yaml
+++ b/legacy/tests/helper/perftest.yaml
diff --git a/tests/data/helper/suite.yaml b/legacy/tests/helper/suite.yaml
index 718ae440..718ae440 100644
--- a/tests/data/helper/suite.yaml
+++ b/legacy/tests/helper/suite.yaml
diff --git a/tests/data/helper/version.yaml b/legacy/tests/helper/version.yaml
index b23f16f2..b23f16f2 100644
--- a/tests/data/helper/version.yaml
+++ b/legacy/tests/helper/version.yaml
diff --git a/tests/unit/utils/spawn_vm_test.py b/legacy/tests/spawn_vm_test.py
index ba237378..ba237378 100644
--- a/tests/unit/utils/spawn_vm_test.py
+++ b/legacy/tests/spawn_vm_test.py
diff --git a/scripts/__init__.py b/legacy/utils/__init__.py
index e69de29b..e69de29b 100644
--- a/scripts/__init__.py
+++ b/legacy/utils/__init__.py
diff --git a/qtip/utils/ansible_api.py b/legacy/utils/ansible_api.py
index 9e1d249e..9e1d249e 100644
--- a/qtip/utils/ansible_api.py
+++ b/legacy/utils/ansible_api.py
diff --git a/qtip/utils/args_handler.py b/legacy/utils/args_handler.py
index 993b1035..993b1035 100644
--- a/qtip/utils/args_handler.py
+++ b/legacy/utils/args_handler.py
diff --git a/qtip/utils/cli.py b/legacy/utils/cli.py
index 5e566f27..5e566f27 100644
--- a/qtip/utils/cli.py
+++ b/legacy/utils/cli.py
diff --git a/qtip/utils/create_zones.py b/legacy/utils/create_zones.py
index 5e378c83..5e378c83 100644
--- a/qtip/utils/create_zones.py
+++ b/legacy/utils/create_zones.py
diff --git a/scripts/ref_results/__init__.py b/legacy/utils/dashboard/__init__.py
index e69de29b..e69de29b 100644
--- a/scripts/ref_results/__init__.py
+++ b/legacy/utils/dashboard/__init__.py
diff --git a/qtip/utils/dashboard/pushtoDB.py b/legacy/utils/dashboard/pushtoDB.py
index 427d39c4..427d39c4 100644
--- a/qtip/utils/dashboard/pushtoDB.py
+++ b/legacy/utils/dashboard/pushtoDB.py
diff --git a/qtip/utils/driver.py b/legacy/utils/driver.py
index 9894e0f5..9894e0f5 100644
--- a/qtip/utils/driver.py
+++ b/legacy/utils/driver.py
diff --git a/qtip/utils/env_setup.py b/legacy/utils/env_setup.py
index 7bbedfcf..7bbedfcf 100644
--- a/qtip/utils/env_setup.py
+++ b/legacy/utils/env_setup.py
diff --git a/tests/functional/__init__.py b/legacy/utils/report/__init__.py
index e69de29b..e69de29b 100644
--- a/tests/functional/__init__.py
+++ b/legacy/utils/report/__init__.py
diff --git a/qtip/utils/report/get_indices.py b/legacy/utils/report/get_indices.py
index 91219c0b..91219c0b 100644
--- a/qtip/utils/report/get_indices.py
+++ b/legacy/utils/report/get_indices.py
diff --git a/qtip/utils/report/get_results.py b/legacy/utils/report/get_results.py
index 23fd5383..23fd5383 100644
--- a/qtip/utils/report/get_results.py
+++ b/legacy/utils/report/get_results.py
diff --git a/qtip/utils/report/qtip_graph.py b/legacy/utils/report/qtip_graph.py
index acbda40c..acbda40c 100644
--- a/qtip/utils/report/qtip_graph.py
+++ b/legacy/utils/report/qtip_graph.py
diff --git a/qtip/utils/report/qtip_report.py b/legacy/utils/report/qtip_report.py
index 6809e892..6809e892 100644
--- a/qtip/utils/report/qtip_report.py
+++ b/legacy/utils/report/qtip_report.py
diff --git a/qtip/utils/spawn_vm.py b/legacy/utils/spawn_vm.py
index f38c9a3a..f38c9a3a 100644
--- a/qtip/utils/spawn_vm.py
+++ b/legacy/utils/spawn_vm.py
diff --git a/tests/unit/api/__init__.py b/legacy/utils/transform/__init__.py
index e69de29b..e69de29b 100644
--- a/tests/unit/api/__init__.py
+++ b/legacy/utils/transform/__init__.py
diff --git a/qtip/utils/transform/dpi_transform.py b/legacy/utils/transform/dpi_transform.py
index ee29d8e2..ee29d8e2 100644
--- a/qtip/utils/transform/dpi_transform.py
+++ b/legacy/utils/transform/dpi_transform.py
diff --git a/qtip/utils/transform/final_report.py b/legacy/utils/transform/final_report.py
index 274742d4..274742d4 100644
--- a/qtip/utils/transform/final_report.py
+++ b/legacy/utils/transform/final_report.py
diff --git a/qtip/utils/transform/fio_transform.py b/legacy/utils/transform/fio_transform.py
index 5ecac823..5ecac823 100755
--- a/qtip/utils/transform/fio_transform.py
+++ b/legacy/utils/transform/fio_transform.py
diff --git a/qtip/utils/transform/iperf_transform.py b/legacy/utils/transform/iperf_transform.py
index b52e4634..b52e4634 100644
--- a/qtip/utils/transform/iperf_transform.py
+++ b/legacy/utils/transform/iperf_transform.py
diff --git a/qtip/utils/transform/ramspeed_transform.py b/legacy/utils/transform/ramspeed_transform.py
index 960f84fc..960f84fc 100644
--- a/qtip/utils/transform/ramspeed_transform.py
+++ b/legacy/utils/transform/ramspeed_transform.py
diff --git a/qtip/utils/transform/ssl_transform.py b/legacy/utils/transform/ssl_transform.py
index de84d24b..de84d24b 100644
--- a/qtip/utils/transform/ssl_transform.py
+++ b/legacy/utils/transform/ssl_transform.py
diff --git a/qtip/utils/transform/ubench_transform.py b/legacy/utils/transform/ubench_transform.py
index ab5fe171..ab5fe171 100644
--- a/qtip/utils/transform/ubench_transform.py
+++ b/legacy/utils/transform/ubench_transform.py
diff --git a/tests/unit/utils/__init__.py b/opt/__init__.py
index e69de29b..e69de29b 100644
--- a/tests/unit/utils/__init__.py
+++ b/opt/__init__.py
diff --git a/opt/servers/ansible.cfg b/opt/infra/ansible.cfg
index 93022c4e..93022c4e 100644
--- a/opt/servers/ansible.cfg
+++ b/opt/infra/ansible.cfg
diff --git a/opt/servers/inventory b/opt/infra/inventory
index 1ba1dc45..1ba1dc45 100644
--- a/opt/servers/inventory
+++ b/opt/infra/inventory
diff --git a/opt/servers/roles/docker/handlers/main.yml b/opt/infra/roles/docker/handlers/main.yml
index 985f8d41..985f8d41 100644
--- a/opt/servers/roles/docker/handlers/main.yml
+++ b/opt/infra/roles/docker/handlers/main.yml
diff --git a/opt/servers/roles/docker/tasks/main.yml b/opt/infra/roles/docker/tasks/main.yml
index 1cf3bea0..1cf3bea0 100644
--- a/opt/servers/roles/docker/tasks/main.yml
+++ b/opt/infra/roles/docker/tasks/main.yml
diff --git a/opt/servers/roles/elk/tasks/main.yml b/opt/infra/roles/elk/tasks/main.yml
index ed987f1a..ed987f1a 100644
--- a/opt/servers/roles/elk/tasks/main.yml
+++ b/opt/infra/roles/elk/tasks/main.yml
diff --git a/opt/servers/roles/mongo/tasks/main.yml b/opt/infra/roles/mongo/tasks/main.yml
index 81fb49b5..81fb49b5 100644
--- a/opt/servers/roles/mongo/tasks/main.yml
+++ b/opt/infra/roles/mongo/tasks/main.yml
diff --git a/opt/servers/roles/nginx/defaults/main.yml b/opt/infra/roles/nginx/defaults/main.yml
index cdd1d773..cdd1d773 100644
--- a/opt/servers/roles/nginx/defaults/main.yml
+++ b/opt/infra/roles/nginx/defaults/main.yml
diff --git a/opt/servers/roles/nginx/handlers/main.yml b/opt/infra/roles/nginx/handlers/main.yml
index 35585363..35585363 100644
--- a/opt/servers/roles/nginx/handlers/main.yml
+++ b/opt/infra/roles/nginx/handlers/main.yml
diff --git a/opt/servers/roles/nginx/tasks/main.yml b/opt/infra/roles/nginx/tasks/main.yml
index 5c78166c..5c78166c 100644
--- a/opt/servers/roles/nginx/tasks/main.yml
+++ b/opt/infra/roles/nginx/tasks/main.yml
diff --git a/opt/servers/roles/nginx/templates/elk.conf.j2 b/opt/infra/roles/nginx/templates/elk.conf.j2
index 271690e0..271690e0 100644
--- a/opt/servers/roles/nginx/templates/elk.conf.j2
+++ b/opt/infra/roles/nginx/templates/elk.conf.j2
diff --git a/opt/servers/roles/nginx/templates/qtip.conf.j2 b/opt/infra/roles/nginx/templates/qtip.conf.j2
index 3a15c6ba..3a15c6ba 100644
--- a/opt/servers/roles/nginx/templates/qtip.conf.j2
+++ b/opt/infra/roles/nginx/templates/qtip.conf.j2
diff --git a/opt/servers/roles/nginx/templates/testapi.conf.j2 b/opt/infra/roles/nginx/templates/testapi.conf.j2
index 6a4d388b..6a4d388b 100644
--- a/opt/servers/roles/nginx/templates/testapi.conf.j2
+++ b/opt/infra/roles/nginx/templates/testapi.conf.j2
diff --git a/opt/servers/roles/qtip/files/run_qtip_server.sh b/opt/infra/roles/qtip/files/run_qtip_server.sh
index 0f5cafea..0f5cafea 100644
--- a/opt/servers/roles/qtip/files/run_qtip_server.sh
+++ b/opt/infra/roles/qtip/files/run_qtip_server.sh
diff --git a/opt/servers/roles/qtip/tasks/main.yml b/opt/infra/roles/qtip/tasks/main.yml
index b8292791..b8292791 100644
--- a/opt/servers/roles/qtip/tasks/main.yml
+++ b/opt/infra/roles/qtip/tasks/main.yml
diff --git a/opt/servers/roles/testapi/files/run_testapi.sh b/opt/infra/roles/testapi/files/run_testapi.sh
index f9ba8387..f9ba8387 100644
--- a/opt/servers/roles/testapi/files/run_testapi.sh
+++ b/opt/infra/roles/testapi/files/run_testapi.sh
diff --git a/opt/servers/roles/testapi/tasks/main.yml b/opt/infra/roles/testapi/tasks/main.yml
index 8ca91396..8ca91396 100644
--- a/opt/servers/roles/testapi/tasks/main.yml
+++ b/opt/infra/roles/testapi/tasks/main.yml
diff --git a/opt/servers/roles/user/defaults/main.yml b/opt/infra/roles/user/defaults/main.yml
index ef496dd4..ef496dd4 100644
--- a/opt/servers/roles/user/defaults/main.yml
+++ b/opt/infra/roles/user/defaults/main.yml
diff --git a/opt/servers/roles/user/files/akhil.authorized_keys b/opt/infra/roles/user/files/akhil.authorized_keys
index 43942621..43942621 100644
--- a/opt/servers/roles/user/files/akhil.authorized_keys
+++ b/opt/infra/roles/user/files/akhil.authorized_keys
diff --git a/opt/servers/roles/user/files/serena.authorized_keys b/opt/infra/roles/user/files/serena.authorized_keys
index 5cdbfe00..5cdbfe00 100644
--- a/opt/servers/roles/user/files/serena.authorized_keys
+++ b/opt/infra/roles/user/files/serena.authorized_keys
diff --git a/opt/servers/roles/user/files/sudoers.d-qtip b/opt/infra/roles/user/files/sudoers.d-qtip
index e96d278a..e96d278a 100644
--- a/opt/servers/roles/user/files/sudoers.d-qtip
+++ b/opt/infra/roles/user/files/sudoers.d-qtip
diff --git a/opt/servers/roles/user/files/taseer.authorized_keys b/opt/infra/roles/user/files/taseer.authorized_keys
index 2efb95e8..2efb95e8 100644
--- a/opt/servers/roles/user/files/taseer.authorized_keys
+++ b/opt/infra/roles/user/files/taseer.authorized_keys
diff --git a/opt/servers/roles/user/files/yujunz.authorized_keys b/opt/infra/roles/user/files/yujunz.authorized_keys
index 83ed540d..83ed540d 100644
--- a/opt/servers/roles/user/files/yujunz.authorized_keys
+++ b/opt/infra/roles/user/files/yujunz.authorized_keys
diff --git a/opt/servers/roles/user/files/zhifeng.authorized_keys b/opt/infra/roles/user/files/zhifeng.authorized_keys
index 195cfdca..195cfdca 100644
--- a/opt/servers/roles/user/files/zhifeng.authorized_keys
+++ b/opt/infra/roles/user/files/zhifeng.authorized_keys
diff --git a/opt/servers/roles/user/tasks/main.yml b/opt/infra/roles/user/tasks/main.yml
index b1b5be9b..b1b5be9b 100644
--- a/opt/servers/roles/user/tasks/main.yml
+++ b/opt/infra/roles/user/tasks/main.yml
diff --git a/opt/servers/test.yml b/opt/infra/test.yml
index ee3ded13..ee3ded13 100644
--- a/opt/servers/test.yml
+++ b/opt/infra/test.yml
diff --git a/opt/plugin/__init__.py b/opt/plugin/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/opt/plugin/__init__.py
diff --git a/opt/plugin/doctor/__init__.py b/opt/plugin/doctor/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/opt/plugin/doctor/__init__.py
diff --git a/opt/plugin/doctor/collector.yaml b/opt/plugin/doctor/collector.yaml
new file mode 100644
index 00000000..40167588
--- /dev/null
+++ b/opt/plugin/doctor/collector.yaml
@@ -0,0 +1,24 @@
+# collector configuration for doctor verification job
+# collect performance data from log files
+collector: logfile
+# transform collected data into timeline
+transformer: timeline
+logfiles:
+ - filename: doctor_consumer.log
+ # 2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63
+ - match: doctor consumer notified at \d+(\.\d+)?$
+ checkpoint: notified consumer
+ - filename: doctor_inspector.log
+ # 2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3
+ # 2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
+ # 2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
+ - match: event posted at \d+(\.\d+)?$
+ checkpoint: posted event
+ - match: doctor mark vm\(.*\) error at \d+(\.\d+)?$
+ checkpoint: marked VM error
+ - match: doctor mark host\(.*\) down at \d+(\.\d+)?$
+ checkpoint: marked host down
+ - filename: disable_network.log
+ # doctor set host down at 1482894965.164096803
+ - match: doctor set host down at \d+(\.\d+)?$
+ - checkpoint: set host down
diff --git a/opt/plugin/doctor/sample/__init__.py b/opt/plugin/doctor/sample/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/opt/plugin/doctor/sample/__init__.py
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log
new file mode 100755
index 00000000..8c577b59
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/consumer.log
@@ -0,0 +1,4 @@
+ * Running on http://0.0.0.0:12346/ (Press CTRL+C to quit)
+2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63
+2016-12-28 03:16:05,630 consumer.py 27 INFO received data = {"severity": "moderate", "alarm_name": "doctor_alarm1", "current": "alarm", "alarm_id": "d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97", "reason": "Event <id=e27a3560-7abd-41e2-9591-b2c15baa3e94,event_type=compute.instance.update> hits the query <query=[{\"field\": \"traits.state\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"error\"}, {\"field\": \"traits.instance_id\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"51c0739a-0b87-4dd7-8d22-ff7fd8af04bb\"}]>.", "reason_data": {"type": "event", "event": {"event_type": "compute.instance.update", "traits": [["state", 1, "error"], ["user_id", 1, "c844f5397a924d908f1246515e4843e6"], ["service", 1, "compute"], ["disk_gb", 2, 1], ["instance_type", 1, "m1.tiny"], ["tenant_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["root_gb", 2, 1], ["ephemeral_gb", 2, 0], ["instance_type_id", 2, 3], ["vcpus", 2, 1], ["memory_mb", 2, 512], ["instance_id", 1, "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb"], ["host", 1, "overcloud-controller-2.ool-virtual1"], ["request_id", 1, "req-531a60d6-ee47-4fb1-8d86-9e42425d6427"], ["project_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["launched_at", 4, "2016-12-28T03:14:43"]], "message_signature": "b784003e35c0ec531c44dc33836b4b4a8d4544351894c12951ef946edf18bb8f", "raw": {}, "generated": "2016-12-28T03:16:05.494044", "message_id": "e27a3560-7abd-41e2-9591-b2c15baa3e94"}}, "previous": "insufficient data"}
+127.0.0.1 - - [28/Dec/2016 03:16:05] "POST /failure HTTP/1.1" 200 -
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log
new file mode 100755
index 00000000..1b488491
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/disable_network.log
@@ -0,0 +1,11 @@
+++ sudo ip a
+++ awk '/ 192.0.2.5\//{print $7}'
++ dev=eth0
++ sleep 1
+++ date +%s.%N
++ echo 'doctor set host down at' 1482894965.164096803
+doctor set host down at 1482894965.164096803
++ sudo ip link set eth0 down
++ sleep 180
++ sudo ip link set eth0 up
++ sleep 1
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log
new file mode 100755
index 00000000..e040e809
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_consumer.log
@@ -0,0 +1,2 @@
+2016-12-28 03:16:05,630 consumer.py 26 INFO doctor consumer notified at 1482894965.63
+2016-12-28 03:16:05,630 consumer.py 27 INFO received data = {"severity": "moderate", "alarm_name": "doctor_alarm1", "current": "alarm", "alarm_id": "d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97", "reason": "Event <id=e27a3560-7abd-41e2-9591-b2c15baa3e94,event_type=compute.instance.update> hits the query <query=[{\"field\": \"traits.state\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"error\"}, {\"field\": \"traits.instance_id\", \"op\": \"eq\", \"type\": \"string\", \"value\": \"51c0739a-0b87-4dd7-8d22-ff7fd8af04bb\"}]>.", "reason_data": {"type": "event", "event": {"event_type": "compute.instance.update", "traits": [["state", 1, "error"], ["user_id", 1, "c844f5397a924d908f1246515e4843e6"], ["service", 1, "compute"], ["disk_gb", 2, 1], ["instance_type", 1, "m1.tiny"], ["tenant_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["root_gb", 2, 1], ["ephemeral_gb", 2, 0], ["instance_type_id", 2, 3], ["vcpus", 2, 1], ["memory_mb", 2, 512], ["instance_id", 1, "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb"], ["host", 1, "overcloud-controller-2.ool-virtual1"], ["request_id", 1, "req-531a60d6-ee47-4fb1-8d86-9e42425d6427"], ["project_id", 1, "dffffadad42d41d5a618570f52e3305f"], ["launched_at", 4, "2016-12-28T03:14:43"]], "message_signature": "b784003e35c0ec531c44dc33836b4b4a8d4544351894c12951ef946edf18bb8f", "raw": {}, "generated": "2016-12-28T03:16:05.494044", "message_id": "e27a3560-7abd-41e2-9591-b2c15baa3e94"}}, "previous": "insufficient data"}
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log
new file mode 100755
index 00000000..e2fd9477
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_inspector.log
@@ -0,0 +1,6 @@
+2016-12-28 03:15:01,868 inspector.py 50 DEBUG get hostname=overcloud-novacompute-1.ool-virtual1 from server=<Server: doctor_vm1>
+2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3
+2016-12-28 03:16:05,299 inspector.py 77 INFO inspector = <__main__.DoctorInspectorSample object at 0x7fcce167aa90>
+2016-12-28 03:16:05,299 inspector.py 78 INFO received data = {"hostname": "overcloud-novacompute-1.ool-virtual1", "type": "compute.host.down"}
+2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
+2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log
new file mode 100755
index 00000000..a4c6ec8f
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/doctor_monitor.log
@@ -0,0 +1,3 @@
+2016-12-28 03:15:00,890 monitor.py 65 DEBUG start ping to host overcloud-novacompute-1.ool-virtual1 (ip=192.0.2.5)
+2016-12-28 03:16:05,292 monitor.py 74 INFO doctor monitor detected at 1482894965.29
+2016-12-28 03:16:05,960 monitor.py 76 INFO ping timeout, quit monitoring...
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log
new file mode 100755
index 00000000..2933ad61
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/functest-doctor.log
@@ -0,0 +1,421 @@
++ IMAGE_URL=https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img
++ IMAGE_NAME=cirros
++ IMAGE_FILE=cirros.img
++ IMAGE_FORMAT=qcow2
++ VM_NAME=doctor_vm1
++ VM_FLAVOR=m1.tiny
++ ALARM_NAME=doctor_alarm1
++ INSPECTOR_PORT=12345
++ CONSUMER_PORT=12346
++ DOCTOR_USER=doctor
++ DOCTOR_PW=doctor
++ DOCTOR_PROJECT=doctor
++ DOCTOR_ROLE=admin
++++ dirname ./run.sh
+++ cd .
+++ pwd
++ TOP_DIR=/home/opnfv/repos/doctor/tests
++ ssh_opts='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
++ as_doctor_user='--os-username doctor --os-password doctor
+ --os-tenant-name doctor'
++ echo 'Note: doctor/tests/run.sh has been executed.'
+Note: doctor/tests/run.sh has been executed.
++ trap cleanup EXIT
++ source /home/opnfv/repos/doctor/tests/functions-common
++ source /home/opnfv/repos/doctor/tests/lib/installer
+++ INSTALLER_TYPE=apex
+++ INSTALLER_IP=192.168.122.70
+++ ssh_opts_cpu='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
++ source /home/opnfv/repos/doctor/tests/lib/inspector
+++ INSPECTOR_TYPE=sample
++ setup_installer
++ is_installer_supported apex
++ local installer=apex
++ [[ -f /home/opnfv/repos/doctor/tests/lib/installers/apex ]]
++ source /home/opnfv/repos/doctor/tests/lib/installers/apex
++ is_set INSTALLER_IP
++ local 'var=$INSTALLER_IP'
++ eval '[ -n "$INSTALLER_IP" ]'
+++ '[' -n 192.168.122.70 ']'
++ installer_get_ssh_keys
++ sudo scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@192.168.122.70:/home/stack/.ssh/id_rsa instack_key
+Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
+++ whoami
+++ whoami
++ sudo chown root:root instack_key
++ chmod 400 instack_key
++ ssh_opts_cpu+=' -i instack_key'
++ installer_apply_patches
++ return
++ echo 'preparing VM image...'
+preparing VM image...
++ download_image
++ use_existing_image=false
++ openstack image list
++ grep -q ' cirros '
++ use_existing_image=true
++ [[ true == false ]]
++ register_image
++ openstack image list
++ grep -q ' cirros '
++ return 0
++ echo 'creating test user...'
+creating test user...
++ create_test_user
++ openstack project list
++ grep -q ' doctor '
++ openstack project create doctor
++-------------+----------------------------------+
+| Field | Value |
++-------------+----------------------------------+
+| description | None |
+| enabled | True |
+| id | dffffadad42d41d5a618570f52e3305f |
+| name | doctor |
++-------------+----------------------------------+
++ openstack user list
++ grep -q ' doctor '
++ openstack user create doctor --password doctor --project doctor
++------------+----------------------------------+
+| Field | Value |
++------------+----------------------------------+
+| email | None |
+| enabled | True |
+| id | c844f5397a924d908f1246515e4843e6 |
+| name | doctor |
+| project_id | dffffadad42d41d5a618570f52e3305f |
+| username | doctor |
++------------+----------------------------------+
++ openstack user role list doctor --project doctor
++ grep -q ' admin '
++ openstack role add admin --user doctor --project doctor
++-----------+----------------------------------+
+| Field | Value |
++-----------+----------------------------------+
+| domain_id | None |
+| id | 4b0d10e07705433c865958471d7c4ff5 |
+| name | admin |
++-----------+----------------------------------+
++ echo 'creating VM...'
+creating VM...
++ boot_vm
++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
++ grep -q ' doctor_vm1 '
++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server create --flavor m1.tiny --image cirros doctor_vm1
++--------------------------------------+------------------------------------------------+
+| Field | Value |
++--------------------------------------+------------------------------------------------+
+| OS-DCF:diskConfig | MANUAL |
+| OS-EXT-AZ:availability_zone | |
+| OS-EXT-SRV-ATTR:host | None |
+| OS-EXT-SRV-ATTR:hypervisor_hostname | None |
+| OS-EXT-SRV-ATTR:instance_name | |
+| OS-EXT-STS:power_state | 0 |
+| OS-EXT-STS:task_state | scheduling |
+| OS-EXT-STS:vm_state | building |
+| OS-SRV-USG:launched_at | None |
+| OS-SRV-USG:terminated_at | None |
+| accessIPv4 | |
+| accessIPv6 | |
+| addresses | |
+| adminPass | jL7t3S3DUrCX |
+| config_drive | |
+| created | 2016-12-28T03:14:37Z |
+| flavor | m1.tiny (7f924f9f-2fa0-40d4-95fb-53079307ec66) |
+| hostId | |
+| id | 51c0739a-0b87-4dd7-8d22-ff7fd8af04bb |
+| image | cirros (3071d9bf-7be5-4cb1-a706-428daa446ec3) |
+| key_name | None |
+| name | doctor_vm1 |
+| os-extended-volumes:volumes_attached | [] |
+| progress | 0 |
+| project_id | dffffadad42d41d5a618570f52e3305f |
+| properties | |
+| security_groups | [{u'name': u'default'}] |
+| status | BUILD |
+| updated | 2016-12-28T03:14:37Z |
+| user_id | c844f5397a924d908f1246515e4843e6 |
++--------------------------------------+------------------------------------------------+
++ sleep 1
++ wait_for_vm_launch
++ echo 'waiting for vm launch...'
+waiting for vm launch...
++ count=0
++ [[ 0 -lt 60 ]]
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
+++ grep ' doctor_vm1 '
+++ awk '{print $6}'
++ state=BUILD
++ [[ BUILD == \A\C\T\I\V\E ]]
++ [[ BUILD == \E\R\R\O\R ]]
++ count=1
++ sleep 1
++ [[ 1 -lt 60 ]]
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
+++ grep ' doctor_vm1 '
+++ awk '{print $6}'
++ state=ACTIVE
++ [[ ACTIVE == \A\C\T\I\V\E ]]
++ sleep 5
++ return 0
++ echo 'get computer host info...'
+get computer host info...
++ get_compute_host_info
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server show doctor_vm1
+++ grep OS-EXT-SRV-ATTR:host
+++ awk '{ print $4 }'
++ COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
++ compute_host_in_undercloud=overcloud-novacompute-1
++ die_if_not_set 45 COMPUTE_HOST 'Failed to get compute hostname'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ is_installer apex
++ local installer=apex
++ [[ apex == apex ]]
++ COMPUTE_USER=heat-admin
+++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192.168.122.70 'source stackrc; nova show overcloud-novacompute-1 | awk '\''/ ctlplane network /{print $5}'\'''
+Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
++ COMPUTE_IP=192.0.2.5
++ die_if_not_set 63 COMPUTE_IP 'Could not resolve overcloud-novacompute-1.ool-virtual1. Either manually set COMPUTE_IP or enable DNS resolution.'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ echo COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
+COMPUTE_HOST=overcloud-novacompute-1.ool-virtual1
++ echo COMPUTE_IP=192.0.2.5
+COMPUTE_IP=192.0.2.5
++ ping -c 1 192.0.2.5
+PING 192.0.2.5 (192.0.2.5) 56(84) bytes of data.
+64 bytes from 192.0.2.5: icmp_seq=1 ttl=63 time=0.275 ms
+
+--- 192.0.2.5 ping statistics ---
+1 packets transmitted, 1 received, 0% packet loss, time 0ms
+rtt min/avg/max/mdev = 0.275/0.275/0.275/0.000 ms
++ [[ 0 -ne 0 ]]
++ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5 exit
+Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
++ [[ 0 -ne 0 ]]
++ echo 'creating alarm...'
+creating alarm...
++ create_alarm
++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-list
++ grep -q ' doctor_alarm1 '
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
+++ grep ' doctor_vm1 '
+++ awk '{print $2}'
++ vm_id=51c0739a-0b87-4dd7-8d22-ff7fd8af04bb
++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-event-create --name doctor_alarm1 --alarm-action http://localhost:12346/failure --description 'VM failure' --enabled True --repeat-actions False --severity moderate --event-type compute.instance.update -q 'traits.state=string::error; traits.instance_id=string::51c0739a-0b87-4dd7-8d22-ff7fd8af04bb'
++---------------------------+----------------------------------------------------------------------+
+| Property | Value |
++---------------------------+----------------------------------------------------------------------+
+| alarm_actions | ["http://localhost:12346/failure"] |
+| alarm_id | d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 |
+| description | VM failure |
+| enabled | True |
+| event_type | compute.instance.update |
+| insufficient_data_actions | [] |
+| name | doctor_alarm1 |
+| ok_actions | [] |
+| project_id | dffffadad42d41d5a618570f52e3305f |
+| query | [{"field": "traits.state", "type": "string", "value": "error", "op": |
+| | "eq"}, {"field": "traits.instance_id", "type": "string", "value": |
+| | "51c0739a-0b87-4dd7-8d22-ff7fd8af04bb", "op": "eq"}] |
+| repeat_actions | False |
+| severity | moderate |
+| state | insufficient data |
+| type | event |
+| user_id | c844f5397a924d908f1246515e4843e6 |
++---------------------------+----------------------------------------------------------------------+
++ echo 'starting doctor sample components...'
+starting doctor sample components...
++ start_inspector
++ is_inspector_supported sample
++ local inspector=sample
++ [[ -f /home/opnfv/repos/doctor/tests/lib/inspectors/sample ]]
++ source /home/opnfv/repos/doctor/tests/lib/inspectors/sample
++ start_inspector_sample
++ pgrep -f 'python inspector.py'
++ start_monitor
++ pgrep -f 'python monitor.py'
++ python inspector.py 12345
++ start_consumer
++ sudo -E python monitor.py overcloud-novacompute-1.ool-virtual1 192.0.2.5 sample http://127.0.0.1:12345/events
++ pgrep -f 'python consumer.py'
++ is_installer local
++ local installer=local
++ python consumer.py 12346
++ [[ local == apex ]]
++ is_installer apex
++ local installer=apex
++ [[ apex == apex ]]
+++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192.168.122.70 'source stackrc; nova list | grep '\'' overcloud-controller-[0-9] '\'' | sed -e '\''s/^.*ctlplane=//'\'' -e '\''s/ *|$//'\'''
+Warning: Permanently added '192.168.122.70' (ECDSA) to the list of known hosts.
++ CONTROLLER_IPS='192.0.2.6
+192.0.2.8
+192.0.2.9'
++ die_if_not_set 185 CONTROLLER_IPS 'Could not get CONTROLLER_IPS.'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600
++ sleep 60
++ sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600
++ echo 'injecting host failure...'
+injecting host failure...
++ inject_failure
++ echo 'disabling network of compute host [overcloud-novacompute-1.ool-virtual1] for 3 mins...'
+disabling network of compute host [overcloud-novacompute-1.ool-virtual1] for 3 mins...
++ cat
++ sed -i -e s/@COMPUTE_IP@/192.0.2.5/ disable_network.sh
++ chmod +x disable_network.sh
++ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key disable_network.sh heat-admin@192.0.2.5:
+Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
++ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5 'nohup ./disable_network.sh > disable_network.log 2>&1 &'
+Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
++ sleep 60
++ check_host_status '(DOWN|UNKNOWN)'
++ expected_state='(DOWN|UNKNOWN)'
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor --os-compute-api-version 2.16 server show doctor_vm1
+++ grep host_status
++ host_status_line='| host_status | DOWN |'
+++ echo '|' host_status '|' DOWN '|'
+++ awk '{print $4}'
++ host_status=DOWN
++ die_if_not_set 283 host_status 'host_status not reported by: nova show doctor_vm1'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ [[ (DOWN|UNKNOWN) =~ DOWN ]]
++ echo 'doctor_vm1 showing host_status: DOWN'
+doctor_vm1 showing host_status: DOWN
++ calculate_notification_time
+++ grep 'doctor monitor detected at' monitor.log
+++ awk '{print $10}'
++ detected=1482894965.29
+++ grep 'doctor consumer notified at' consumer.log
+++ awk '{print $10}'
++ notified=1482894965.63
++ grep -q 'doctor consumer notified at' consumer.log
++ [[ PROFILER == \p\o\c ]]
++ echo '1482894965.63 1482894965.29'
++ awk '{
+ d = $1 - $2;
+ if (d < 1 && d > 0) { print d " OK"; exit 0 }
+ else { print d " NG"; exit 1 }
+ }'
+0.34 OK
++ echo done
+done
++ cleanup
++ set +e
++ echo cleanup...
+cleanup...
++ stop_monitor
++ pgrep -f 'python monitor.py'
++ return 0
++ stop_inspector
++ stop_inspector_sample
++ pgrep -f 'python inspector.py'
+377
+++ pgrep -f 'python inspector.py'
++ kill 377
++ stop_consumer
++ pgrep -f 'python consumer.py'
+381
+++ pgrep -f 'python consumer.py'
++ kill 381
++ is_installer local
++ local installer=local
++ [[ local == apex ]]
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
+++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.6 -R 12346:localhost:12346 sleep 600'
++ kill 398
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
+./run.sh: line 206: 377 Terminated python inspector.py "$INSPECTOR_PORT" > inspector.log 2>&1
+./run.sh: line 206: 381 Terminated python consumer.py "$CONSUMER_PORT" > consumer.log 2>&1
+++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.8 -R 12346:localhost:12346 sleep 600'
++ kill 399
++ for ip in '$CONTROLLER_IPS'
++ forward_rule='-R 12346:localhost:12346'
++ tunnel_command='sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
+++ pgrep -f 'sudo ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.9 -R 12346:localhost:12346 sleep 600'
++ kill 400
++ echo 'waiting disabled compute host back to be enabled...'
+waiting disabled compute host back to be enabled...
++ python ./nova_force_down.py overcloud-novacompute-1.ool-virtual1 --unset
+<Response [200]>
++ sleep 240
++ check_host_status UP
++ expected_state=UP
+++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor --os-compute-api-version 2.16 server show doctor_vm1
+++ grep host_status
++ host_status_line='| host_status | UP |'
+++ echo '|' host_status '|' UP '|'
+++ awk '{print $4}'
++ host_status=UP
++ die_if_not_set 283 host_status 'host_status not reported by: nova show doctor_vm1'
++ local exitcode=0
++ local xtrace
+++ set +o
+++ grep xtrace
++ xtrace='set -o xtrace'
++ set +o xtrace
++ [[ UP =~ UP ]]
++ echo 'doctor_vm1 showing host_status: UP'
+doctor_vm1 showing host_status: UP
++ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i instack_key heat-admin@192.0.2.5:disable_network.log .
+Warning: Permanently added '192.0.2.5' (ECDSA) to the list of known hosts.
++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server list
++ grep -q ' doctor_vm1 '
++ openstack --os-username doctor --os-password doctor --os-tenant-name doctor server delete doctor_vm1
++ sleep 1
+++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-list
+++ grep ' doctor_alarm1 '
+++ awk '{print $2}'
++ alarm_id=d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97
++ sleep 1
++ '[' -n d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97 ']'
++ ceilometer --os-username doctor --os-password doctor --os-tenant-name doctor alarm-delete d90f1c1e-e4cb-4d35-a35a-9ced6bbfee97
++ sleep 1
+++ openstack image list
+++ grep ' cirros '
+++ awk '{print $2}'
++ image_id=3071d9bf-7be5-4cb1-a706-428daa446ec3
++ sleep 1
++ [[ true == false ]]
++ openstack role remove admin --user doctor --project doctor
++ openstack project delete doctor
++ openstack user delete doctor
++ cleanup_installer
++ cleanup_installer_apex
++ return
++ cleanup_inspector
++ cleanup_inspector_sample
++ return
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log
new file mode 100755
index 00000000..0d07de02
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/inspector.log
@@ -0,0 +1,9 @@
+2016-12-28 03:15:01,868 inspector.py 50 DEBUG get hostname=overcloud-novacompute-1.ool-virtual1 from server=<Server: doctor_vm1>
+ * Running on http://127.0.0.1:12345/ (Press CTRL+C to quit)
+2016-12-28 03:16:05,299 inspector.py 76 INFO event posted at 1482894965.3
+2016-12-28 03:16:05,299 inspector.py 77 INFO inspector = <__main__.DoctorInspectorSample object at 0x7fcce167aa90>
+2016-12-28 03:16:05,299 inspector.py 78 INFO received data = {"hostname": "overcloud-novacompute-1.ool-virtual1", "type": "compute.host.down"}
+2016-12-28 03:16:05,299 inspector.py 56 INFO doctor mark vm(<Server: doctor_vm1>) error at 1482894965.3
+2016-12-28 03:16:05,506 inspector.py 66 INFO doctor mark host(overcloud-novacompute-1.ool-virtual1) down at 1482894965.51
+<Response [200]>
+127.0.0.1 - - [28/Dec/2016 03:16:05] "POST /events HTTP/1.1" 200 -
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log
new file mode 100755
index 00000000..a4c6ec8f
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/monitor.log
@@ -0,0 +1,3 @@
+2016-12-28 03:15:00,890 monitor.py 65 DEBUG start ping to host overcloud-novacompute-1.ool-virtual1 (ip=192.0.2.5)
+2016-12-28 03:16:05,292 monitor.py 74 INFO doctor monitor detected at 1482894965.29
+2016-12-28 03:16:05,960 monitor.py 76 INFO ping timeout, quit monitoring...
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log
new file mode 100755
index 00000000..70cc8945
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.6.log
@@ -0,0 +1 @@
+Warning: Permanently added '192.0.2.6' (ECDSA) to the list of known hosts.
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log
new file mode 100755
index 00000000..54222d2a
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.8.log
@@ -0,0 +1 @@
+Warning: Permanently added '192.0.2.8' (ECDSA) to the list of known hosts.
diff --git a/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log
new file mode 100755
index 00000000..2660035b
--- /dev/null
+++ b/opt/plugin/doctor/sample/doctor-verify-apex-sample-master/ssh_tunnel.192.0.2.9.log
@@ -0,0 +1 @@
+Warning: Permanently added '192.0.2.9' (ECDSA) to the list of known hosts.
diff --git a/qtip/api/cmd/server.py b/qtip/api/cmd/server.py
index 3b89af1e..eea45ad3 100644
--- a/qtip/api/cmd/server.py
+++ b/qtip/api/cmd/server.py
@@ -10,7 +10,8 @@
from flask import Flask
from flask_restful import Api
from flask_restful_swagger import swagger
-import qtip.api.router.mapper as mapper
+
+import legacy.api.router.mapper as mapper
app = Flask(__name__)
api = swagger.docs(Api(app), apiVersion='0.1', description='QTIP API specs')
diff --git a/qtip/base/benchmark.py b/qtip/base/benchmark.py
deleted file mode 100644
index b38e6016..00000000
--- a/qtip/base/benchmark.py
+++ /dev/null
@@ -1,95 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from itertools import chain
-from os import listdir
-from os import path
-import yaml
-
-
-class Property(object):
- # list
- NAME = 'name'
- CONTENT = 'content'
- ABSPATH = 'abspath'
- # content
- TITLE = 'title'
- DESCRIPTION = 'description'
- # spec
- ALGORITHM = 'algorithm'
- SECTIONS = 'sections'
- WEIGHT = 'weight'
- METRICS = 'metrics'
- SPEC = 'spec'
- WORKLOADS = 'workloads'
- # plan
- INFO = 'info'
- FACILITY = 'facility'
- ENGINEER = 'engineer'
- SUITES = 'suites'
- # suite
- QPI_SPEC = 'QPI_spec'
- CONDITION = 'condition'
- CASES = 'cases'
- # case
- METRIC_SPEC = 'metric_spec'
- CONFIG = 'config'
-
-
-class QtipError(Exception):
- pass
-
-
-class Algorithm(object):
- ARITHMETIC_MEAN = 'arithmetic mean'
- WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
- GEOMETRIC_MEAN = 'geometric mean'
- WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
-
-
-ROOT_DIR = 'benchmarks'
-
-
-class Benchmark(object):
- """Abstract class of QTIP benchmarks"""
- DEFAULT_DIR = '.'
- _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
- ROOT_DIR)]
-
- def __init__(self, name, paths=None):
- self._file = name
- self._abspath = self._find(name, paths)
- self.name = path.splitext(name)[0]
-
- def _find(self, name, paths):
- """find a benchmark in searching paths"""
- paths = self._paths if paths is None else paths
- name = path.join(self.DEFAULT_DIR, name)
- for p in paths:
- abspath = path.join(p, name)
- if path.exists(abspath):
- return abspath
- raise QtipError("'{}' not found in paths: {}".format(name, paths))
-
- @classmethod
- def list_all(cls, paths=None):
- """list all available benchmarks"""
- paths = cls._paths if paths is None else paths
- names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR))
- for p in paths])
- for name in names:
- item = cls(name, paths=paths)
- yield {
- Property.NAME: name,
- Property.ABSPATH: item._abspath,
- Property.CONTENT: item.content()}
-
- def content(self):
- """description of benchmark"""
- return yaml.safe_load(file(self._abspath))
diff --git a/qtip/base/constant.py b/qtip/base/constant.py
new file mode 100644
index 00000000..76481b47
--- /dev/null
+++ b/qtip/base/constant.py
@@ -0,0 +1,50 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class FormulaName(object):
+ """formula names"""
+ ARITHMETIC_MEAN = 'arithmetic mean'
+ WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
+ GEOMETRIC_MEAN = 'geometric mean'
+ WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
+
+
+class PkgName(object):
+ """QTIP package names"""
+ COLLECTOR = 'collector'
+ DRIVER = 'driver'
+ REPORTER = 'reporter'
+ RUNNER = 'runner'
+ SPEC = 'spec'
+
+
+class PropName(object):
+ """property names"""
+ # list
+ NAME = 'name'
+ CONTENT = 'content'
+ ABSPATH = 'abspath'
+ # content
+ name = 'name'
+ DESCRIPTION = 'description'
+ # spec
+ SECTIONS = 'sections'
+ WEIGHT = 'weight'
+ FORMULA = 'formula'
+ METRICS = 'metrics'
+ WORKLOADS = 'workloads'
+ # plan
+ CONFIG = 'config'
+ FACILITY = 'facility'
+ ENGINEER = 'engineer'
+ DRIVER = 'driver'
+ COLLECTOR = 'collector'
+ REPORTER = 'reporter'
+ QPIS = 'QPIs'
diff --git a/qtip/base/error.py b/qtip/base/error.py
new file mode 100644
index 00000000..01a7f7a6
--- /dev/null
+++ b/qtip/base/error.py
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class QtipError(Exception):
+ pass
+
+
+class InvalidFormat(QtipError):
+ def __init__(self, filename):
+ self.filename = filename
+
+
+class NotFound(QtipError):
+ def __init__(self, module, package='qtip'):
+ self.package = package
+ self.module = module
+
+
+class ToBeDoneError(QtipError):
+ """something still to be done"""
+ def __init__(self, method, module):
+ self.method = method
+ self.module = module
+
+
+def make_tbd(method, module='qtip'):
+ def tbd():
+ raise ToBeDoneError(method, module)
+ return tbd
diff --git a/qtip/runner/case.py b/qtip/cli/commands/cmd_metric.py
index eb3febc2..7f5c18d3 100644
--- a/qtip/runner/case.py
+++ b/qtip/cli/commands/cmd_metric.py
@@ -7,11 +7,27 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
+import click
+from qtip.cli.entry import Context
-class Case(object):
- def __init__(self, spec, paths=None):
- self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths)
- self.config = spec[Property.CONFIG]
+pass_context = click.make_pass_decorator(Context, ensure=False)
+
+
+@click.group()
+@pass_context
+def cli(ctx):
+ ''' Performance Metrics Group '''
+ pass
+
+
+@cli.command('list', help='List all the Metric Groups')
+@pass_context
+def cmd_list(ctx):
+ pass
+
+
+@cli.command('run', help='Run tests to collect Performance Metrics')
+@pass_context
+def cmd_run(ctx):
+ pass
diff --git a/qtip/cli/commands/cmd_perftest.py b/qtip/cli/commands/cmd_perftest.py
deleted file mode 100644
index 129ee2d9..00000000
--- a/qtip/cli/commands/cmd_perftest.py
+++ /dev/null
@@ -1,58 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from prettytable import PrettyTable
-import yaml
-import click
-import os
-from qtip.cli import helper
-
-
-class PerfTest:
-
- def __init__(self):
- self.path = os.path.join(helper.fetch_root(), 'perftest/summary')
-
- def list(self):
- table = PrettyTable(["Name", "Description"])
- table.align = 'l'
- with open(self.path) as tests:
- line = tests.read()
- data = yaml.safe_load(line)['test_cases']
- for i in range(0, len(data)):
- points = data[i]
- table.add_row([points['name'], points['description']])
- click.echo(table)
-
- def run(self):
- click.echo("Run a perftest")
-
-
-@click.group()
-def cli():
- pass
-
-
-@cli.group()
-@click.pass_context
-def perftest(ctx):
- pass
-
-
-_perftest = PerfTest()
-
-
-@perftest.command("list", help="Lists all perftest benchmarks.")
-def list():
- _perftest.list()
-
-
-@perftest.command("run", help="Executes a single perftest benchmark.")
-def execute():
- _perftest.run()
diff --git a/qtip/cli/commands/cmd_plan.py b/qtip/cli/commands/cmd_plan.py
index 01bf8251..6f622e5a 100644
--- a/qtip/cli/commands/cmd_plan.py
+++ b/qtip/cli/commands/cmd_plan.py
@@ -9,7 +9,8 @@
import click
from prettytable import PrettyTable
-from qtip.runner.plan import Plan
+
+from qtip.loader.plan import Plan
@click.group()
diff --git a/qtip/collector/__init__.py b/qtip/collector/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/collector/__init__.py
diff --git a/qtip/drivers/base.py b/qtip/collector/base.py
index 1aa8d8ad..cd8fc797 100644
--- a/qtip/drivers/base.py
+++ b/qtip/collector/base.py
@@ -8,5 +8,5 @@
##############################################################################
-class BaseDriver(object):
- """performance testing tool driver"""
+class BaseCollector(object):
+ """performance metrics collector"""
diff --git a/qtip/spec/qpi.py b/qtip/collector/logfile.py
index b7d7aa02..6528ea9f 100644
--- a/qtip/spec/qpi.py
+++ b/qtip/collector/logfile.py
@@ -7,12 +7,8 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark
+from base import BaseCollector
-class QPISpec(Benchmark):
- """
- a QPI specification defines how to calculate a performance index from
- collected metrics.
- """
- DEFAULT_DIR = 'QPI'
+class LogfileCollector(BaseCollector):
+ """collect performance metrics from log files"""
diff --git a/qtip/collector/transformer/__init__.py b/qtip/collector/transformer/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/collector/transformer/__init__.py
diff --git a/qtip/agent/collector.py b/qtip/collector/transformer/base.py
index 3df21379..bf38c493 100644
--- a/qtip/agent/collector.py
+++ b/qtip/collector/transformer/base.py
@@ -8,6 +8,5 @@
##############################################################################
-class Collector(object):
- """collect test result and test condition"""
- pass
+class BaseTransformer(object):
+ """Transform collected performance metrics into required format"""
diff --git a/qtip/collector/transformer/timeline.py b/qtip/collector/transformer/timeline.py
new file mode 100644
index 00000000..9dd5850c
--- /dev/null
+++ b/qtip/collector/transformer/timeline.py
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from base import BaseTransformer
+
+
+class Timeline(BaseTransformer):
+ """transform collected metrics into object required by timeline template"""
diff --git a/qtip/driver/__init__.py b/qtip/driver/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/driver/__init__.py
diff --git a/qtip/drivers/ansible.py b/qtip/driver/ansible.py
index 04e9f9bd..04e9f9bd 100644
--- a/qtip/drivers/ansible.py
+++ b/qtip/driver/ansible.py
diff --git a/qtip/driver/base.py b/qtip/driver/base.py
new file mode 100644
index 00000000..6f5cab3c
--- /dev/null
+++ b/qtip/driver/base.py
@@ -0,0 +1,20 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseDriver(object):
+ """performance testing tool driver"""
+ def pre_run(self):
+ pass
+
+ def run(self):
+ pass
+
+ def post_run(self):
+ pass
diff --git a/qtip/driver/sample.py b/qtip/driver/sample.py
new file mode 100644
index 00000000..9b347949
--- /dev/null
+++ b/qtip/driver/sample.py
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from base import BaseDriver
+
+
+class SampleDriver(BaseDriver):
+ """sample driver that generates random data for testing"""
diff --git a/qtip/drivers/yardstick.py b/qtip/driver/yardstick.py
index 83f1b3d8..83f1b3d8 100644
--- a/qtip/drivers/yardstick.py
+++ b/qtip/driver/yardstick.py
diff --git a/qtip/loader/__init__.py b/qtip/loader/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/loader/__init__.py
diff --git a/qtip/loader/base.py b/qtip/loader/base.py
new file mode 100644
index 00000000..d3a7e1d3
--- /dev/null
+++ b/qtip/loader/base.py
@@ -0,0 +1,62 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from itertools import chain
+from os import listdir
+from os import path
+import yaml
+
+from qtip.base.error import InvalidFormat, NotFound
+from qtip.base.constant import PropName
+
+
+ROOT_DIR = path.join(path.dirname(__file__), path.pardir, path.pardir,
+ 'benchmarks')
+
+
+class BaseLoader(object):
+ """Abstract class of QTIP benchmark loader"""
+ RELATIVE_PATH = '.'
+ _paths = [ROOT_DIR]
+
+ def __init__(self, name, paths=None):
+ self._file = name
+ self._abspath = self._find(name, paths=paths)
+
+ try:
+ content = yaml.safe_load(file(self._abspath))
+ except yaml.YAMLError:
+ # TODO(yujunz) log yaml error
+ raise InvalidFormat(self._abspath)
+
+ self.name = content[PropName.NAME] if PropName.NAME in content \
+ else path.splitext(name)[0]
+ self.content = content
+
+ def _find(self, name, paths=None):
+ """find a benchmark in searching paths"""
+ paths = self._paths if paths is None else paths
+ for p in paths:
+ abspath = path.join(p, self.RELATIVE_PATH, name)
+ if path.exists(abspath):
+ return abspath
+ raise NotFound(name, paths)
+
+ @classmethod
+ def list_all(cls, paths=None):
+ """list all available benchmarks"""
+ paths = cls._paths if paths is None else paths
+ names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH))
+ for p in paths])
+ for name in names:
+ item = cls(name, paths=paths)
+ yield {
+ PropName.NAME: name,
+ PropName.ABSPATH: item._abspath,
+ PropName.CONTENT: item.content}
diff --git a/qtip/spec/metric.py b/qtip/loader/metric.py
index e9c70547..8b6fa5d3 100644
--- a/qtip/spec/metric.py
+++ b/qtip/loader/metric.py
@@ -7,10 +7,10 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark
+from base import BaseLoader
-class MetricSpec(Benchmark):
+class MetricSpec(BaseLoader):
"""metrics in QTIP are categorized by performance test tools, such as
dhrystone, whetstone and etc"""
- DEFAULT_DIR = 'metrics'
+ RELATIVE_PATH = 'metric'
diff --git a/qtip/runner/plan.py b/qtip/loader/plan.py
index 265ad8d7..cf517ea2 100644
--- a/qtip/runner/plan.py
+++ b/qtip/loader/plan.py
@@ -7,20 +7,21 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark, Property
-from qtip.runner.suite import Suite
+from qtip.base.constant import PropName
+from qtip.loader.base import BaseLoader
+from qtip.loader.qpi import QPISpec
-class Plan(Benchmark):
+
+class Plan(BaseLoader):
+ """
+ a benchmark plan is consist of configuration and a QPI list
"""
- a benchmark plan is consist of basic information and several suites"""
- DEFAULT_DIR = 'plans'
+ RELATIVE_PATH = 'plan'
def __init__(self, name, paths=None):
- super(Plan, self).__init__(name, paths=paths)
- content = self.content()
+ super(Plan, self).__init__(name, paths)
- self.info = content[Property.INFO]
- self.suites = [Suite(suite, paths=paths)
- for suite in content[Property.SUITES]]
+ self.qpis = [QPISpec(qpi, paths=paths)
+ for qpi in self.content[PropName.QPIS]]
diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py
new file mode 100644
index 00000000..fe2323dd
--- /dev/null
+++ b/qtip/loader/qpi.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from base import BaseLoader
+from metric import MetricSpec
+
+from qtip.base.constant import PropName
+from qtip.util.formula import Formula
+
+
+class QPISpec(BaseLoader):
+ """
+ a QPI specification defines how to calculate a performance index from
+ collected metrics.
+ """
+ RELATIVE_PATH = 'QPI'
+
+ def __init__(self, name, paths=None):
+ super(QPISpec, self).__init__(name, paths=paths)
+ content = self.content
+ self.formula = Formula(content[PropName.FORMULA])
+ self.sections = [Section(record, paths=paths)
+ for record in content[PropName.SECTIONS]]
+
+
+class Section(object):
+ def __init__(self, content, paths=None):
+ self.name = content[PropName.NAME]
+ self.weight = content[PropName.WEIGHT]
+ self.formula = Formula(content[PropName.FORMULA])
+ self.metrics = [MetricSpec(record, paths=paths)
+ for record in content[PropName.METRICS]]
diff --git a/qtip/reporter/__init__.py b/qtip/reporter/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/reporter/__init__.py
diff --git a/qtip/reporter/base.py b/qtip/reporter/base.py
new file mode 100644
index 00000000..b931d14d
--- /dev/null
+++ b/qtip/reporter/base.py
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseReporter(object):
+ """benchmark result reporter"""
+ def __init__(self, collector=None):
+ self.collector = collector
diff --git a/qtip/agent/reporter.py b/qtip/reporter/console/__init__.py
index b5c4acfa..02d63288 100644
--- a/qtip/agent/reporter.py
+++ b/qtip/reporter/console/__init__.py
@@ -8,6 +8,8 @@
##############################################################################
-class Reporter(object):
- """generate test report and push test data to database"""
- pass
+from qtip.reporter.base import BaseReporter
+
+
+class ConsoleReporter(BaseReporter):
+ """report result to console"""
diff --git a/qtip/reporter/console/timeline.j2 b/qtip/reporter/console/timeline.j2
new file mode 100644
index 00000000..9c18a996
--- /dev/null
+++ b/qtip/reporter/console/timeline.j2
@@ -0,0 +1,8 @@
+{% title %}
+{% for phase in phases %}
+{{ phase.name|upper }}{{ "TIME" }}
+{% for cp in phase.checkpoints %}
+{{ cp.name }}{{ cp.timestamp}}
+{% endfor %}
+{% endfor %}
+Total: {{ total }}
diff --git a/qtip/runner/__init__.py b/qtip/runner/__init__.py
index e69de29b..eab81156 100644
--- a/qtip/runner/__init__.py
+++ b/qtip/runner/__init__.py
@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.constant import PkgName, PropName
+from qtip.base.error import NotFound
+from qtip.collector.stdout import StdoutCollector
+from qtip.driver.random import RandomDriver
+from qtip.reporter.console import ConsoleReporter
+
+
+class Runner(object):
+ def __init__(self, spec, config=None):
+ if config is None:
+ config = spec[PropName.CONFIG]
+
+ driver_name = config[PropName.DRIVER]
+ collector_name = config[PropName.COLLECTOR]
+ reporter_name = config[PropName.REPORTER]
+
+ # TODO(yujunz) dynamically load modules by name
+
+ if driver_name == 'random':
+ self.driver = RandomDriver()
+ else:
+ raise NotFound(driver_name, package=PkgName.DRIVER)
+
+ if collector_name == 'stdout':
+ self.collector = StdoutCollector()
+ else:
+ raise NotFound(collector_name,
+ package=PkgName.COLLECTOR)
+
+ if reporter_name == 'console':
+ self.reporter = ConsoleReporter()
+ else:
+ raise NotFound(reporter_name,
+ package=PkgName.REPORTER)
diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py
deleted file mode 100644
index 55033d2b..00000000
--- a/qtip/runner/suite.py
+++ /dev/null
@@ -1,22 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.benchmark import Property
-from qtip.spec.qpi import QPISpec
-from qtip.runner.case import Case
-
-
-class Suite(object):
- """a suite of benchmark cases under specified condition"""
- def __init__(self, spec, paths=None):
- self._paths = paths
- self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths)
- self.condition = spec.get(Property.CONDITION, {})
- self.cases = [Case(case_spec, paths)
- for case_spec in spec.get(Property.CASES, [])]
diff --git a/qtip/util/__init__.py b/qtip/util/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/qtip/util/__init__.py
diff --git a/qtip/util/formula.py b/qtip/util/formula.py
new file mode 100644
index 00000000..cdfbae86
--- /dev/null
+++ b/qtip/util/formula.py
@@ -0,0 +1,29 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import numpy
+
+from qtip.base.error import make_tbd
+from qtip.base.constant import FormulaName
+
+
+MAPPING = {
+ FormulaName.ARITHMETIC_MEAN: numpy.mean,
+ FormulaName.WEIGHTED_ARITHMETIC_MEAN: numpy.average,
+ # TODO(yujunz) find or implement the method
+ FormulaName.GEOMETRIC_MEAN: make_tbd(FormulaName.GEOMETRIC_MEAN, __name__),
+ # TODO(yujunz) find or implement the method
+ FormulaName.WEIGHTED_GEOMETRIC_MEAN:
+ make_tbd(FormulaName.GEOMETRIC_MEAN, __name__)}
+
+
+class Formula:
+ """calculate a score from give data"""
+ def __init__(self, name):
+ self.calculate = MAPPING[name]
diff --git a/qtip/utils/logger_utils.py b/qtip/util/logger.py
index d5e76a64..d5e76a64 100644
--- a/qtip/utils/logger_utils.py
+++ b/qtip/util/logger.py
diff --git a/requirements.txt b/requirements.txt
index cdaeef70..ec566b83 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,4 @@ Flask==0.11.1
Flask-RESTful==0.3.5
flask-restful-swagger==0.19
ansible==2.1.1.0
+numpy==1.11.3
diff --git a/setup.cfg b/setup.cfg
index 790a365b..ced6d22e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -20,4 +20,7 @@ console_scripts =
packages =
qtip
data_files =
- etc/qtip = benchmarks/*
+ opt/qtip = opt/*
+
+[tool:pytest]
+addopts = --ignore=legacy
diff --git a/tests/conftest.py b/tests/conftest.py
index 8e2ecf7c..7acb75e6 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,8 +8,11 @@
##############################################################################
from os import path
+
import pytest
+from qtip.loader.plan import Plan
+
@pytest.fixture(scope='session')
def data_root():
@@ -19,3 +22,8 @@ def data_root():
@pytest.fixture(scope='session')
def benchmarks_root(data_root):
return path.join(data_root, 'benchmarks')
+
+
+@pytest.fixture(scope='session')
+def plan(benchmarks_root):
+ return Plan('fake-plan.yaml', [benchmarks_root])
diff --git a/tests/data/benchmarks/QPI/compute.yaml b/tests/data/benchmarks/QPI/compute.yaml
index 5ad8a4b6..e28efaf9 100644
--- a/tests/data/benchmarks/QPI/compute.yaml
+++ b/tests/data/benchmarks/QPI/compute.yaml
@@ -1,44 +1,29 @@
title: compute
description: sample performance index of computing
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
sections:
- name: Integer
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dhrystone.yaml
- workloads:
- - single_cpu
- - multi_cpu
+ - dhrystone.yaml
- name: Float
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dhrystone.yaml
- workloads:
- - single_cpu
- - multi_cpu
+ - whetstone.yaml
- name: Memory
weight: 0.2
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: ramspeed.yaml
- workloads:
- - int: [add, average, copy, scale, triad]
- - float: [add, average, copy, scale, triad]
+ - ramspeed.yaml
- name: DPI
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dpi.yaml
- workloads:
- - bps
- - pps
+ - dpi.yaml
- name: SSL
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: ssl.yaml
- workloads:
- - aes_128_cbc: [512, 1024, 2048, 4096]
- - rsa_sig: [16, 64, 256, 1024, 8192]
+ - ssl.yaml
diff --git a/tests/data/benchmarks/QPI/fake-qpi.yaml b/tests/data/benchmarks/QPI/fake-qpi.yaml
new file mode 100644
index 00000000..aa1097f4
--- /dev/null
+++ b/tests/data/benchmarks/QPI/fake-qpi.yaml
@@ -0,0 +1,9 @@
+name: Fake QPI
+description: a fake QPI producing random result
+formula: weighted arithmetic mean
+sections:
+- name: Fake Section
+ weight: 0.5
+ formula: geometric mean
+ metrics:
+ - fake-metric.yaml
diff --git a/tests/data/benchmarks/metric/dhrystone.yaml b/tests/data/benchmarks/metric/dhrystone.yaml
new file mode 100644
index 00000000..220b7841
--- /dev/null
+++ b/tests/data/benchmarks/metric/dhrystone.yaml
@@ -0,0 +1,9 @@
+name: dhrystone
+description: >
+ A synthetic computing benchmark program intended to be representative of
+ system (integer) programming.
+links:
+ - https://en.wikipedia.org/wiki/Dhrystone
+workloads:
+ - single_cpu
+ - multi_cpu
diff --git a/tests/data/benchmarks/metrics/dpi.yaml b/tests/data/benchmarks/metric/dpi.yaml
index fc24c8d1..fc24c8d1 100644
--- a/tests/data/benchmarks/metrics/dpi.yaml
+++ b/tests/data/benchmarks/metric/dpi.yaml
diff --git a/tests/data/benchmarks/metric/fake-metric.yaml b/tests/data/benchmarks/metric/fake-metric.yaml
new file mode 100644
index 00000000..c5fcc5c2
--- /dev/null
+++ b/tests/data/benchmarks/metric/fake-metric.yaml
@@ -0,0 +1,7 @@
+name: fake compute
+description: >
+ a fake compute performance index which generate random result
+workloads: # all supported workloads
+ - fake_workload_1
+ - fake_workload_2
+ - fake_workload_3
diff --git a/tests/data/benchmarks/metrics/ramspeed.yaml b/tests/data/benchmarks/metric/ramspeed.yaml
index bb7618b7..bb7618b7 100644
--- a/tests/data/benchmarks/metrics/ramspeed.yaml
+++ b/tests/data/benchmarks/metric/ramspeed.yaml
diff --git a/tests/data/benchmarks/metrics/ssl.yaml b/tests/data/benchmarks/metric/ssl.yaml
index 21e8add7..21e8add7 100644
--- a/tests/data/benchmarks/metrics/ssl.yaml
+++ b/tests/data/benchmarks/metric/ssl.yaml
diff --git a/tests/data/benchmarks/metric/whetstone.yaml b/tests/data/benchmarks/metric/whetstone.yaml
new file mode 100644
index 00000000..448c9645
--- /dev/null
+++ b/tests/data/benchmarks/metric/whetstone.yaml
@@ -0,0 +1,10 @@
+name: whetstone
+description: >
+ A synthetic benchmark for evaluating the performance of computers.
+ The Whetstone benchmark primarily measures the floating-point arithmetic
+ performance.
+links:
+ - https://en.wikipedia.org/wiki/Whetstone_(benchmark)
+workloads:
+ - single_cpu
+ - multi_cpu
diff --git a/tests/data/benchmarks/metrics/dhrystone.yaml b/tests/data/benchmarks/metrics/dhrystone.yaml
deleted file mode 100644
index b0d55ed2..00000000
--- a/tests/data/benchmarks/metrics/dhrystone.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-name: dhrystone
-description: >
- a synthetic computing benchmark program intended to be representative of
- system (integer) programming
-workloads:
- - single_cpu
- - multi_cpu
diff --git a/tests/data/benchmarks/metrics/whetstone.yaml b/tests/data/benchmarks/metrics/whetstone.yaml
deleted file mode 100644
index d83680c4..00000000
--- a/tests/data/benchmarks/metrics/whetstone.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-name: dhrystone
-description: a synthetic benchmark for evaluating the performance of computers
-workloads:
- - single_cpu
- - multi_cpu
diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml
new file mode 100644
index 00000000..8887f66d
--- /dev/null
+++ b/tests/data/benchmarks/plan/fake-plan.yaml
@@ -0,0 +1,10 @@
+name: fake plan
+description: fake benchmark plan for demonstration and testing
+config:
+ facility: local
+ engineer: local
+ driver: sample
+ collector: logfile
+ reporter: console
+QPIs:
+ - fake-qpi.yaml
diff --git a/tests/data/benchmarks/plans/verification.yaml b/tests/data/benchmarks/plans/verification.yaml
deleted file mode 100644
index b146ee37..00000000
--- a/tests/data/benchmarks/plans/verification.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-title: verification
-description: benchmark plan for QTIP verification
-info:
- facility: opnfv-ci
- engineer: opnfv-bot
-suites:
- - QPI_spec: compute.yaml
-# Uncomment next line to setup suite running condition
-# condition: {}
-# Uncomment the following lines setup case running configuration
-# cases:
-# - metric_spec: dhrystone.yaml
-# config: {}
-# - metric_spec: whetstone.yaml
-# config: {}
-# - metric_spec: ramspeed.yaml
-# config: {}
-# - metric_spec: dpi.yaml
-# config: {}
-# - metric_spec: ssl.yaml
-# config: {}
diff --git a/tests/unit/runner/case_test.py b/tests/unit/cli/test_metric.py
index 59a54a84..ed10689c 100644
--- a/tests/unit/runner/case_test.py
+++ b/tests/unit/cli/test_metric.py
@@ -7,9 +7,22 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.spec.metric import MetricSpec
+import pytest
+from click.testing import CliRunner
+from qtip.cli.commands.cmd_metric import cli
-def init_test(case):
- assert isinstance(case.metric_spec, MetricSpec)
- assert isinstance(case.config, dict)
+
+@pytest.fixture()
+def runner():
+ return CliRunner()
+
+
+def test_list(runner):
+ result = runner.invoke(cli, ['list'])
+ assert result.output == ''
+
+
+def test_run(runner):
+ result = runner.invoke(cli, ['run'])
+ assert result.output == ''
diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py
index 5eced700..d2be0388 100644
--- a/tests/unit/loader/metric_test.py
+++ b/tests/unit/loader/metric_test.py
@@ -9,8 +9,8 @@
import pytest
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
+from qtip.base.constant import PropName
+from qtip.loader.metric import MetricSpec
@pytest.fixture(scope='module')
@@ -29,17 +29,17 @@ def init_test(metric_spec):
def list_all_test():
metric_list = MetricSpec.list_all()
- assert len(list(metric_list)) is 1
+ assert len(list(metric_list)) is 6
for desc in metric_list:
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
+ assert PropName.NAME in desc
+ assert PropName.DESCRIPTION in desc
+ assert PropName.ABSPATH in desc
+ assert PropName.ABSPATH is not None
def content_test(metric):
- content = metric.content()
- assert Property.NAME in content
- assert Property.DESCRIPTION in content
- assert Property.WORKLOADS in content
- assert isinstance(content[Property.WORKLOADS], list)
+ content = metric.content
+ assert PropName.NAME in content
+ assert PropName.DESCRIPTION in content
+ assert PropName.WORKLOADS in content
+ assert isinstance(content[PropName.WORKLOADS], list)
diff --git a/tests/unit/runner/plan_test.py b/tests/unit/loader/plan_test.py
index d783e5e9..6aab5e8a 100644
--- a/tests/unit/runner/plan_test.py
+++ b/tests/unit/loader/plan_test.py
@@ -9,12 +9,15 @@
import pytest
-from qtip.base.benchmark import Property
-from qtip.runner.plan import Plan
+from qtip.base.constant import PropName
+from qtip.loader.plan import Plan, QPISpec
def test_init(plan):
- assert plan.name == 'verification'
+ assert plan.name == 'fake plan'
+ assert isinstance(plan.content, dict)
+ for qpi in plan.qpis:
+ assert isinstance(qpi, QPISpec)
with pytest.raises(TypeError) as excinfo:
Plan()
@@ -26,13 +29,15 @@ def test_list_all(benchmarks_root):
plan_list = Plan.list_all(paths=[benchmarks_root])
assert len(list(plan_list)) is 1
for desc in plan_list:
- assert Property.NAME in desc
- assert Property.CONTENT in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
+ assert PropName.NAME in desc
+ assert PropName.CONTENT in desc
+ assert PropName.ABSPATH in desc
+ assert PropName.ABSPATH is not None
def test_content(plan):
- content = plan.content()
- assert Property.TITLE in content
- assert Property.DESCRIPTION in content
+ content = plan.content
+ assert PropName.NAME in content
+ assert PropName.DESCRIPTION in content
+ assert PropName.CONFIG in content
+ assert PropName.QPIS in content
diff --git a/tests/unit/loader/qpi_test.py b/tests/unit/loader/qpi_test.py
index bfa1f580..4b3fd4d0 100644
--- a/tests/unit/loader/qpi_test.py
+++ b/tests/unit/loader/qpi_test.py
@@ -9,8 +9,8 @@
import pytest
-from qtip.base.benchmark import Algorithm, Property
-from qtip.spec.qpi import QPISpec
+from qtip.base.constant import FormulaName, PropName
+from qtip.loader.qpi import QPISpec
QPI_SPEC = 'compute.yaml'
@@ -31,22 +31,22 @@ def test_init(qpi_spec):
def test_list_all(benchmarks_root):
qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root])
- assert len(list(qpi_spec_list)) is 1
+ assert len(list(qpi_spec_list)) is 2
for item in qpi_spec_list:
- assert Property.NAME in item
- assert Property.CONTENT in item
- assert Property.ABSPATH in item
- assert Property.ABSPATH is not None
+ assert PropName.NAME in item
+ assert PropName.CONTENT in item
+ assert PropName.ABSPATH in item
+ assert PropName.ABSPATH is not None
def test_content(qpi_spec):
- content = qpi_spec.content()
- assert Property.DESCRIPTION in content
- assert Property.ALGORITHM in content
- assert Property.SECTIONS in content
+ content = qpi_spec.content
+ assert PropName.DESCRIPTION in content
+ assert PropName.FORMULA in content
+ assert PropName.SECTIONS in content
- assert content[Property.ALGORITHM] in Algorithm.__dict__.values()
- sections = content[Property.SECTIONS]
+ assert content[PropName.FORMULA] in FormulaName.__dict__.values()
+ sections = content[PropName.SECTIONS]
assert isinstance(sections, list)
for section in sections:
- assert Property.NAME in section
+ assert PropName.NAME in section
diff --git a/tests/unit/runner/conftest.py b/tests/unit/runner/conftest.py
deleted file mode 100644
index 6d14f7ae..00000000
--- a/tests/unit/runner/conftest.py
+++ /dev/null
@@ -1,30 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.base.benchmark import Property
-from qtip.runner.case import Case
-from qtip.runner.plan import Plan
-from qtip.runner.suite import Suite
-
-
-@pytest.fixture(scope='module')
-def plan(benchmarks_root):
- return Plan('verification.yaml', paths=[benchmarks_root])
-
-
-@pytest.fixture(scope='module')
-def suite(plan):
- return Suite(plan[Property.SUITES][0])
-
-
-@pytest.fixture(scope='module')
-def case(suite):
- return Case(suite[Property.CASES][0])
diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py
deleted file mode 100644
index 7dad8f62..00000000
--- a/tests/unit/runner/suite_test.py
+++ /dev/null
@@ -1,27 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.runner.suite import Suite
-from qtip.runner.case import Case
-from qtip.spec.qpi import QPISpec
-
-
-def init_test(suite):
- assert isinstance(suite.qpi, QPISpec)
- assert isinstance(suite.condition, dict)
- assert isinstance(suite.cases, list)
- for case in suite.cases:
- assert isinstance(case, Case)
-
- with pytest.raises(TypeError) as excinfo:
- Suite()
- assert '__init__() takes exactly 2 arguments (1 given)' \
- in str(excinfo.value)
diff --git a/tox.ini b/tox.ini
index 7d17af65..03283e42 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@
# and then run "tox" from this directory.
[tox]
-envlist = py27
+envlist = py27,pep8
skipsdist = True
[testenv]
@@ -32,4 +32,4 @@ commands = flake8 {toxinidir}
show-source = True
ignore = E123,E125,H803,E501
builtins = _
-exclude = build,dist,doc,.eggs,.git,.tox,.venv
+exclude = build,dist,doc,legacy,.eggs,.git,.tox,.venv