diff options
27 files changed, 308 insertions, 144 deletions
@@ -1,71 +1,6 @@ -QTIP Benchmark Suite ---------------------- +QTIP is a performance benchmark service for OPNFV platform -QTIP is a benchmarking suite intended to benchmark the following components of the OPNFV Platform: +QTIP aims to benchmark OPNFV platforms through a "Bottom up" approach. Emphasis +on platform performance through quantitative benchmarks rather than validation. -1. Computing components -2. Networking components -3. Storage components - -The efforts in QTIP are mostly focused on identifying - -1. Benchmarks to run -2. Test cases in which these benchmarks to run -3. Automation of suite to run benchmarks within different test cases -4. Collection of test results - -QTIP Framework can now be called: (qtip.py). - -The Framework can run 5 computing benchmarks: - -1. Dhrystone -2. Whetstone -3. RamBandwidth -4. SSL -5. nDPI - -These benchmarks can be run in 2 test cases: - -1. VM vs Baremetal -2. Baremetal vs Baremetal - -Instructions to run the script: - -1. Download and source the OpenStack `adminrc` file for the deployment on which you want to create the VM for benchmarking -2. run `python qtip.py -s {SUITE} -b {BENCHMARK}` -3. run `python qtip.py -h` for more help -4. list of benchmarks can be found in the `qtip/test_cases` directory -5. SUITE refers to compute, network or storage - -Requirements: - -1. Ansible 1.9.2 -2. Python 2.7 -3. PyYAML - -Configuring Test Cases: - -Test cases can be found within the `test_cases` directory. -For each Test case, a Config.yaml file contains the details for the machines upon which the benchmarks would run. -Edit the IP and the Password fields within the files for the machines on which the benchmark is to run. -A robust framework that would allow to include more tests would be included within the future. - -Jump Host requirements: - -The following packages should be installed on the server from which you intend to run QTIP. - -1: Heat Client -2: Glance Client -3: Nova Client -4: Neutron Client -5: wget -6: PyYaml - -Networking - -1: The Host Machines/compute nodes to be benchmarked should have public/access network -2: The Host Machines/compute nodes should allow Password Login - -QTIP support for Foreman - -{TBA} +See [project wiki](https://wiki.opnfv.org/display/qtip) for more information. diff --git a/benchmarks/perftest/dhrystone.yaml b/benchmarks/perftest/dhrystone.yaml index 478cd970..83de5103 100644 --- a/benchmarks/perftest/dhrystone.yaml +++ b/benchmarks/perftest/dhrystone.yaml @@ -4,10 +4,10 @@ tasks: - name: making dhrystone directory - file: path={{workingdir}}/{{Dest_dir}}/dhrystone state=directory + file: path={{Dest_dir}}/dhrystone state=directory - name: making temporary dhrystone directory - file: path={{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp state=directory + file: path={{Dest_dir}}/dhrystone/dhrystone_temp state=directory - hosts: "{{role}}" become: yes @@ -76,7 +76,7 @@ register: files_to_copy - name: copy results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/dhrystone/dhrystone_temp with_items: "{{files_to_copy.stdout_lines}}" - name: registering log files @@ -84,7 +84,7 @@ register: copy_log_results - name: copying log results - fetch: src={{home_dir.stdout}}/qtip_result/log/{{item}} dest={{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp + fetch: src={{home_dir.stdout}}/qtip_result/log/{{item}} dest={{Dest_dir}}/dhrystone/dhrystone_temp with_items: "{{copy_log_results.stdout_lines}}" - name: cleaning tempT @@ -99,13 +99,13 @@ tasks: - name: extracting_json - shell: ( find {{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/dhrystone/) + shell: (find {{Dest_dir}}/dhrystone/dhrystone_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/dhrystone/) - name: making_logs_folder - file: path={{workingdir}}/{{Dest_dir}}/dhrystone/logs state=directory + file: path={{Dest_dir}}/dhrystone/logs state=directory - name: extracting_log - shell: ( find {{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/dhrystone/logs) + shell: (find {{Dest_dir}}/dhrystone/dhrystone_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/dhrystone/logs) - name: removing dhrystone_temp - file: path={{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp state=directory + file: path={{Dest_dir}}/dhrystone/dhrystone_temp state=directory diff --git a/benchmarks/perftest/dpi.yaml b/benchmarks/perftest/dpi.yaml index 455c5ac9..0afd4f81 100644 --- a/benchmarks/perftest/dpi.yaml +++ b/benchmarks/perftest/dpi.yaml @@ -4,10 +4,10 @@ tasks: - name: making dpi directory - file: path={{workingdir}}/{{Dest_dir}}/dpi state=directory + file: path={{Dest_dir}}/dpi state=directory - name: making temporary whetstone directory - file: path={{workingdir}}/{{Dest_dir}}/dpi/dpi_temp state=directory + file: path={{Dest_dir}}/dpi/dpi_temp state=directory - hosts: "{{role}}" become: yes @@ -91,7 +91,7 @@ register: files_to_copy - name: copy results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/dpi/dpi_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/dpi/dpi_temp with_items: "{{files_to_copy.stdout_lines}}" - name: registering log files @@ -99,7 +99,7 @@ register: copy_log_results - name: copying log results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/dpi/dpi_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/dpi/dpi_temp with_items: "{{copy_log_results.stdout_lines}}" - name: cleaning tempD @@ -114,13 +114,13 @@ tasks: - name: extracting_json - shell: ( find {{workingdir}}/{{Dest_dir}}/dpi/dpi_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/dpi/) + shell: (find {{Dest_dir}}/dpi/dpi_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/dpi/) - name: making_logs_folder - file: path={{workingdir}}/{{Dest_dir}}/dpi/logs state=directory + file: path={{Dest_dir}}/dpi/logs state=directory - name: extracting_log - shell: ( find {{workingdir}}/{{Dest_dir}}/dpi/dpi_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/dpi/logs) + shell: (find {{Dest_dir}}/dpi/dpi_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/dpi/logs) - name: removing dpi_temp - file: path={{workingdir}}/{{Dest_dir}}/dpi/dpi_temp state=absent + file: path={{Dest_dir}}/dpi/dpi_temp state=absent diff --git a/benchmarks/perftest/fio.yaml b/benchmarks/perftest/fio.yaml index 23f2b202..c9fd9011 100644 --- a/benchmarks/perftest/fio.yaml +++ b/benchmarks/perftest/fio.yaml @@ -4,10 +4,10 @@ tasks: - name: making fio directory - file: path={{workingdir}}/{{Dest_dir}}/fio state=directory + file: path={{Dest_dir}}/fio state=directory - name: making temporary fio directory - file: path={{workingdir}}/{{Dest_dir}}/fio/fio_temp state=directory + file: path={{Dest_dir}}/fio/fio_temp state=directory - hosts: "{{role}}" @@ -77,7 +77,7 @@ register: files_to_copy - name: copy results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/fio/fio_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/fio/fio_temp with_items: "{{files_to_copy.stdout_lines}}" - name: registering log files @@ -85,7 +85,7 @@ register: copy_log_results - name: copying log results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/fio/fio_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/fio/fio_temp with_items: "{{copy_log_results.stdout_lines}}" - name: cleaning fio @@ -100,13 +100,13 @@ tasks: - name: extracting_json - shell: ( find {{workingdir}}/{{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/fio/) + shell: (find {{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/fio/) - name: making_logs_folder - file: path={{workingdir}}/{{Dest_dir}}/fio/logs state=directory + file: path={{Dest_dir}}/fio/logs state=directory - name: extracting_log - shell: ( find {{workingdir}}/{{Dest_dir}}/fio/fio_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/fio/logs) + shell: (find {{Dest_dir}}/fio/fio_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/fio/logs) - name: removing fio_log - file: path={{workingdir}}/{{Dest_dir}}/fio/fio_temp state=absent + file: path={{Dest_dir}}/fio/fio_temp state=absent diff --git a/benchmarks/perftest/iperf.yaml b/benchmarks/perftest/iperf.yaml index 46c8d6cf..7886e8a1 100644 --- a/benchmarks/perftest/iperf.yaml +++ b/benchmarks/perftest/iperf.yaml @@ -3,15 +3,11 @@ gather_facts: no tasks: - - name: getting directory - shell: sudo echo $PWD - register: qtip_dir - - name: making Iperf directory - file: path={{workingdir}}/{{Dest_dir}}/iperf state=directory + file: path={{Dest_dir}}/iperf state=directory - name: making temporary iperf directory - file: path={{workingdir}}/{{Dest_dir}}/iperf/iperf_temp state=directory + file: path={{Dest_dir}}/iperf/iperf_temp state=directory - hosts: "{{role}}" @@ -117,7 +113,7 @@ when: rolename =="2-host" and "{{ip2}}" == '' - name: copy results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/iperf/iperf_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/iperf/iperf_temp with_items: "{{files_to_copy.stdout_lines}}" when: rolename =="2-host" and "{{ip2}}" == '' @@ -127,7 +123,7 @@ when: rolename =="2-host" and "{{ip2}}" == '' - name: copying log results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/iperf/iperf_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/iperf/iperf_temp with_items: "{{copy_log_results.stdout_lines}}" when: rolename =="2-host" and "{{ip2}}" == '' @@ -148,19 +144,19 @@ when: role is defined - name: extracting_json - shell: ( find {{workingdir}}/{{Dest_dir}}/iperf/iperf_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/iperf/) + shell: (find {{Dest_dir}}/iperf/iperf_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/iperf/) when: rolename == "2-host" - name: making_logs_folder - file: path={{workingdir}}/{{Dest_dir}}/iperf/logs state=directory + file: path={{Dest_dir}}/iperf/logs state=directory - name: extracting_log - shell: ( find {{workingdir}}/{{Dest_dir}}/iperf/iperf_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/iperf/logs) + shell: ( find {{Dest_dir}}/iperf/iperf_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/iperf/logs) when: rolename == "2-host" - name: removing iperf_raw file - file: path={{workingdir}}/{{Dest_dir}}/iperf/iperf_raw.json state=absent + file: path={{Dest_dir}}/iperf/iperf_raw.json state=absent when: rolename == "2-host" - name: removing iperf_temp - file: path={{workingdir}}/{{Dest_dir}}/iperf/iperf_temp state=absent + file: path={{Dest_dir}}/iperf/iperf_temp state=absent diff --git a/benchmarks/perftest/ramspeed.yaml b/benchmarks/perftest/ramspeed.yaml index 847a8641..f3205755 100644 --- a/benchmarks/perftest/ramspeed.yaml +++ b/benchmarks/perftest/ramspeed.yaml @@ -4,10 +4,10 @@ tasks: - name: making ramspeed directory - file: path={{workingdir}}/{{Dest_dir}}/ramspeed state=directory + file: path={{Dest_dir}}/ramspeed state=directory - name: making temporary ramspeed directory - file: path={{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp state=directory + file: path={{Dest_dir}}/ramspeed/ramspeed_temp state=directory - hosts: "{{role}}" @@ -80,7 +80,7 @@ register: files_to_copy - name: copy results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/ramspeed/ramspeed_temp with_items: "{{files_to_copy.stdout_lines}}" - name: registering log files @@ -88,7 +88,7 @@ register: copy_log_results - name: copying log results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/ramspeed/ramspeed_temp with_items: "{{copy_log_results.stdout_lines}}" - name: cleaning ramspeed directory @@ -103,13 +103,13 @@ tasks: - name: extracting_json - shell: ( find {{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/ramspeed/) + shell: (find /{{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/ramspeed/) - name: making_logs_folder - file: path={{workingdir}}/{{Dest_dir}}/ramspeed/logs state=directory + file: path={{Dest_dir}}/ramspeed/logs state=directory - name: extracting_log - shell: ( find {{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/ramspeed/logs) + shell: ( find {{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/ramspeed/logs) - name: removing ramspeed_log - file: path={{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp state=absent + file: path={{Dest_dir}}/ramspeed/ramspeed_temp state=absent diff --git a/benchmarks/perftest/ssl.yaml b/benchmarks/perftest/ssl.yaml index 0c0e5129..f72c7cfe 100644 --- a/benchmarks/perftest/ssl.yaml +++ b/benchmarks/perftest/ssl.yaml @@ -4,10 +4,10 @@ tasks: - name: making ssl directory - file: path={{workingdir}}/{{Dest_dir}}/ssl state=directory + file: path={{Dest_dir}}/ssl state=directory - name: making temporary ssl directory - file: path={{workingdir}}/{{Dest_dir}}/ssl/ssl_temp state=directory + file: path={{Dest_dir}}/ssl/ssl_temp state=directory - hosts: "{{role}}" become: yes @@ -81,7 +81,7 @@ register: files_to_copy - name: copy results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/ssl/ssl_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/ssl/ssl_temp with_items: "{{files_to_copy.stdout_lines}}" - name: registering log files @@ -89,7 +89,7 @@ register: copy_log_results - name: copying log results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/ssl/ssl_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/ssl/ssl_temp with_items: "{{copy_log_results.stdout_lines}}" - name: cleaning Open_SSL directory @@ -107,13 +107,13 @@ shell: echo $PWD - name: extracting_json - shell: ( find {{workingdir}}/{{Dest_dir}}/ssl/ssl_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/ssl/) + shell: (find {{Dest_dir}}/ssl/ssl_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/ssl/) - name: making_logs_folder - file: path={{workingdir}}/{{Dest_dir}}/ssl/logs state=directory + file: path={{Dest_dir}}/ssl/logs state=directory - name: extracting_log - shell: ( find {{workingdir}}/{{Dest_dir}}/ssl/ssl_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/ssl/logs) + shell: (find {{Dest_dir}}/ssl/ssl_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/ssl/logs) - name: removing ssl_temp - file: path={{workingdir}}/{{Dest_dir}}/ssl/ssl_temp state=absent + file: path={{Dest_dir}}/ssl/ssl_temp state=absent diff --git a/benchmarks/perftest/whetstone.yaml b/benchmarks/perftest/whetstone.yaml index 94532841..0bacb253 100644 --- a/benchmarks/perftest/whetstone.yaml +++ b/benchmarks/perftest/whetstone.yaml @@ -4,10 +4,10 @@ tasks: - name: making whetstone directory - file: path={{workingdir}}/{{Dest_dir}}/whetstone state=directory + file: path={{Dest_dir}}/whetstone state=directory - name: making temporary whetstone directory - file: path={{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp state=directory + file: path={{Dest_dir}}/whetstone/whetstone_temp state=directory - hosts: "{{role}}" become: yes @@ -76,7 +76,7 @@ register: files_to_copy - name: copy results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/whetstone/whetstone_temp with_items: "{{files_to_copy.stdout_lines}}" - name: registering log files @@ -84,7 +84,7 @@ register: copy_log_results - name: copying log results - fetch: src={{home_dir.stdout}}/qtip_result/log/{{item}} dest={{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp + fetch: src={{home_dir.stdout}}/qtip_result/log/{{item}} dest={{Dest_dir}}/whetstone/whetstone_temp with_items: "{{copy_log_results.stdout_lines}}" - name: cleaning tempT directory @@ -99,13 +99,13 @@ tasks: - name: extracting_json - shell: ( find {{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/whetstone/) + shell: (find {{Dest_dir}}/whetstone/whetstone_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/whetstone/) - name: making_logs_folder - file: path={{workingdir}}/{{Dest_dir}}/whetstone/logs state=directory + file: path={{Dest_dir}}/whetstone/logs state=directory - name: extracting_log - shell: ( find {{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/whetstone/logs) + shell: (find {{Dest_dir}}/whetstone/whetstone_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/whetstone/logs) - name: removing whetstone_temp - file: path={{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp state=absent + file: path={{Dest_dir}}/whetstone/whetstone_temp state=absent diff --git a/cli/commands/perftest.py b/cli/commands/perftest.py index c163070a..0eb6d062 100644 --- a/cli/commands/perftest.py +++ b/cli/commands/perftest.py @@ -10,12 +10,14 @@ from prettytable import PrettyTable import yaml import click +import os +from cli import helper class PerfTest: def __init__(self): - self.path = 'benchmarks/perftest/summary' + self.path = os.path.join(helper.fetch_root(), 'perftest/summary') def list(self): table = PrettyTable(["Name", "Description"]) diff --git a/cli/helper.py b/cli/helper.py new file mode 100644 index 00000000..a5865bce --- /dev/null +++ b/cli/helper.py @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os + + +def fetch_root(): + return os.path.join(os.path.dirname(__file__), os.pardir, 'benchmarks/') diff --git a/docker/Dockerfile b/docker/Dockerfile index 8d951fc5..5762409a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,6 +48,8 @@ RUN apt-get install ansible --force-yes -y RUN mkdir -p ${REPOS_DIR} RUN mkdir -p /root/.ssh RUN mkdir -p $HOME/qtip/logs +RUN mkdir -p $HOME/qtip/results + RUN chmod 700 /root/.ssh #Config ansible diff --git a/docs/apidocs/qtip_restful_api.rst b/docs/apidocs/qtip_restful_api.rst index 3f3297d5..06c01292 100644 --- a/docs/apidocs/qtip_restful_api.rst +++ b/docs/apidocs/qtip_restful_api.rst @@ -7,4 +7,4 @@ Qtip restful api **************** -You can get all the Qtip restful api by http://qtip_server_ip:5000/api/spec.html. +You can get all the Qtip restful api by http://restful_api.qtip.openzero.net/api/spec.html. diff --git a/docs/designspec/dashboard.rst b/docs/designspec/dashboard.rst index ad5520b6..60c4720d 100644 --- a/docs/designspec/dashboard.rst +++ b/docs/designspec/dashboard.rst @@ -57,14 +57,95 @@ The condition of a benchmark result includes Conditions that do NOT have an obvious affect on the test result may be ignored, e.g. temperature, power supply. -Deviation ---------- - -Performance tests are usually repeated many times to reduce random disturbance. -This view shall show an overview of deviation among different runs. +Stats +----- + +Performance tests are actually measurement of specific metrics. All measurement +comes with uncertainty. The final result is normally one or a group of metrics +calculated from many repeats. + +For each metric, the stats board shall consist of a diagram of all measured +values and a box of stats:: + + ^ +------------+ + | | count: ? | + | |average: ? | + | | min: ? | + | X | max: ? | + | XXXX XXXX X XXXXX | | + |X XX XX XX XXX XXX XX | | + | XXXXXX X XXXXX XX | | + | | | + | | | + | | | + | | | + | | | + +---------------------------------------------> +------------+ + +The type of diagram and selection of stats shall depend on what metric to show. Comparison ---------- Comparison can be done between different PODs or different configuration on the same PODs. + +In a comparison view, the metrics are displayed in the same diagram. And the +parameters are listed side by side. + +Both common parameters and different parameters are listed. Common values are +merged to the same cell. And user may configure the view to hide common rows. + +A draft design is as following:: + + ^ + | + | + | + | XXXXXXXX + | XXX XX+-+ XXXXXXXXXX + | XXX +XXXX XXXXX + +-+XX X +--+ ++ XXXXXX +-+ + | X+-+X +----+ +-+ +----+X + |X +--+ +---+ XXXXXX X + | +-------+ X + | + | + +-----------------------------------------------------> + + +--------------------+----------------+---------------+ + | different param 1 | | | + | | | | + +-----------------------------------------------------+ + | different param 2 | | | + | | | | + +-------------------------------------+---------------+ + | common param 1 | | + | | | + +-------------------------------------+---------------+ + | different param 3 | | | + | | | | + +-------------------------------------+---------------+ + | common param 2 | | + | | | + +--------------------+--------------------------------+ + +------------+ + | HIDE COMMON| + +------------+ + +Time line +--------- + +Time line diagram for analysis of time critical performance test:: + + +-----------------+-----------+-------------+-------------+-----+ + | | | | | | + +-----------------> | | | | + | +-----------> | | | + | ? ms +-------------> | | + | ? ms +------------>+ | + | ? ms ? ms | + | | + +---------------------------------------------------------------+ + +The time cost between checkpoints shall be displayed in the diagram. diff --git a/docs/releasenotes/brahmaputra.rst b/docs/releasenotes/brahmaputra.rst new file mode 100644 index 00000000..92fafd80 --- /dev/null +++ b/docs/releasenotes/brahmaputra.rst @@ -0,0 +1,78 @@ +*********** +Brahmaputra +*********** + +NOTE: The release note for OPNFV Brahmaputra is missing. This is a copy of the +README. + +QTIP Benchmark Suite +==================== + +QTIP is a benchmarking suite intended to benchmark the following components of the OPNFV Platform: + +1. Computing components +2. Networking components +3. Storage components + +The efforts in QTIP are mostly focused on identifying + +1. Benchmarks to run +2. Test cases in which these benchmarks to run +3. Automation of suite to run benchmarks within different test cases +4. Collection of test results + +QTIP Framework can now be called: (qtip.py). + +The Framework can run 5 computing benchmarks: + +1. Dhrystone +2. Whetstone +3. RamBandwidth +4. SSL +5. nDPI + +These benchmarks can be run in 2 test cases: + +1. VM vs Baremetal +2. Baremetal vs Baremetal + +Instructions to run the script: + +1. Download and source the OpenStack `adminrc` file for the deployment on which you want to create the VM for benchmarking +2. run `python qtip.py -s {SUITE} -b {BENCHMARK}` +3. run `python qtip.py -h` for more help +4. list of benchmarks can be found in the `qtip/test_cases` directory +5. SUITE refers to compute, network or storage + +Requirements: + +1. Ansible 1.9.2 +2. Python 2.7 +3. PyYAML + +Configuring Test Cases: + +Test cases can be found within the `test_cases` directory. +For each Test case, a Config.yaml file contains the details for the machines upon which the benchmarks would run. +Edit the IP and the Password fields within the files for the machines on which the benchmark is to run. +A robust framework that would allow to include more tests would be included within the future. + +Jump Host requirements: + +The following packages should be installed on the server from which you intend to run QTIP. + +1: Heat Client +2: Glance Client +3: Nova Client +4: Neutron Client +5: wget +6: PyYaml + +Networking + +1: The Host Machines/compute nodes to be benchmarked should have public/access network +2: The Host Machines/compute nodes should allow Password Login + +QTIP support for Foreman + +{TBA} diff --git a/docs/releasenotes/index.rst b/docs/releasenotes/index.rst new file mode 100644 index 00000000..5d045388 --- /dev/null +++ b/docs/releasenotes/index.rst @@ -0,0 +1,14 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + + +################## +QTIP Release Notes +################## + +.. toctree:: + :maxdepth: 2 + + brahmaputra.rst diff --git a/func/driver.py b/func/driver.py index 47d00f1f..c765728b 100644 --- a/func/driver.py +++ b/func/driver.py @@ -9,6 +9,7 @@ from utils import logger_utils from operator import add from ansible_api import AnsibleApi +from os.path import expanduser logger = logger_utils.QtipLogger('driver').get @@ -35,7 +36,7 @@ class Driver: def get_common_var_json(self, installer_type, pwd, benchmark_fname, benchmark_detail, pip_dict, proxy_info): - common_json = {'Dest_dir': 'results', + common_json = {'Dest_dir': expanduser('~') + '/qtip/results', 'ip1': '', 'ip2': '', 'installer': str(installer_type), diff --git a/results/foo.txt b/results/foo.txt deleted file mode 100644 index 257cc564..00000000 --- a/results/foo.txt +++ /dev/null @@ -1 +0,0 @@ -foo diff --git a/scripts/ref_results/suite_result.py b/scripts/ref_results/suite_result.py index 4d9eae08..217181f9 100644 --- a/scripts/ref_results/suite_result.py +++ b/scripts/ref_results/suite_result.py @@ -10,6 +10,7 @@ import json import importlib import sys from utils import logger_utils +from os.path import expanduser logger = logger_utils.QtipLogger('suite_result').get @@ -43,7 +44,8 @@ def get_suite_result(suite_name): suite_index = temp / l suite_dict_f = {'index': suite_index, 'suite_results': suite_dict} - with open('results/{0}_result.json'.format(suite_name), 'w+') as result_json: + result_path = expanduser('~') + '/qtip/results' + with open('{0}/{1}_result.json'.format(result_path, suite_name), 'w+') as result_json: json.dump(suite_dict_f, result_json, indent=4, sort_keys=True) return True @@ -9,7 +9,7 @@ setup( description='Platform Performance Benchmarking for OPNFV', author='OPNFV', author_email='zhang.yujunz@zte.com.cn', - install_requires=['click'], + install_requires=['click', 'pyyaml', 'prettytable'], packages=['cli'], entry_points={ 'console_scripts': ['qtip=cli.entry:cli'] diff --git a/supporting/servers/inventory b/supporting/servers/inventory index 1414fa7c..74c04c42 100644 --- a/supporting/servers/inventory +++ b/supporting/servers/inventory @@ -5,3 +5,6 @@ qtip-dev [ngnix-servers] qtip-dev + +[qtip-servers] +qtip-dev diff --git a/supporting/servers/qtip.yml b/supporting/servers/qtip.yml new file mode 100644 index 00000000..f0ac3c4e --- /dev/null +++ b/supporting/servers/qtip.yml @@ -0,0 +1,4 @@ +--- +- hosts: qtip-servers + roles: + - qtip diff --git a/supporting/servers/roles/ngnix/defaults/main.yml b/supporting/servers/roles/ngnix/defaults/main.yml index 855fc7ea..cb3b3934 100644 --- a/supporting/servers/roles/ngnix/defaults/main.yml +++ b/supporting/servers/roles/ngnix/defaults/main.yml @@ -2,3 +2,6 @@ services: - { name: kibana, upstream: 'http://127.0.0.1:5601' } - { name: elastic, upstream: 'http://127.0.0.1:9200' } + +qtip_services: + - { name: restful_api, upstream: 'http://127.0.0.1:5000' } diff --git a/supporting/servers/roles/ngnix/tasks/main.yml b/supporting/servers/roles/ngnix/tasks/main.yml index 8673b3d2..ec146850 100644 --- a/supporting/servers/roles/ngnix/tasks/main.yml +++ b/supporting/servers/roles/ngnix/tasks/main.yml @@ -7,5 +7,6 @@ template: src={{ item }}.conf.j2 dest=/etc/nginx/sites-enabled/{{ item }}.conf with_items: - elk + - qtip notify: - restart nginx diff --git a/supporting/servers/roles/ngnix/templates/qtip.conf.j2 b/supporting/servers/roles/ngnix/templates/qtip.conf.j2 new file mode 100644 index 00000000..3a15c6ba --- /dev/null +++ b/supporting/servers/roles/ngnix/templates/qtip.conf.j2 @@ -0,0 +1,17 @@ +# {{ ansible_managed }} + +# servers +# +{% for service in qtip_services %} +server { + listen 80; + listen 443 ssl; + server_name {{ service.name }}.qtip.openzero.net; + location / { + proxy_pass {{ service.upstream }}; + sub_filter {{ service.upstream }} 'http://{{ service.name }}.qtip.openzero.net'; + sub_filter_once off; + sub_filter_types text/html application/json; + } +} +{% endfor %} diff --git a/supporting/servers/roles/qtip/files/run_qtip_server.sh b/supporting/servers/roles/qtip/files/run_qtip_server.sh new file mode 100644 index 00000000..0f5cafea --- /dev/null +++ b/supporting/servers/roles/qtip/files/run_qtip_server.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +envs="INSTALLER_TYPE=fuel -e INSTALLER_IP=10.20.0.2 -e NODE_NAME=zte-pod1" +docker run --name qtip -id -e $envs -p 5000:5000 opnfv/qtip diff --git a/supporting/servers/roles/qtip/tasks/main.yml b/supporting/servers/roles/qtip/tasks/main.yml new file mode 100644 index 00000000..b8292791 --- /dev/null +++ b/supporting/servers/roles/qtip/tasks/main.yml @@ -0,0 +1,5 @@ +--- +# TODO:change the script to docker_container. +- name: setting up qtip + become: true + script: ../files/run_qtip_server.sh diff --git a/tests/driver_test.py b/tests/driver_test.py index bc705dad..41a7dee4 100644 --- a/tests/driver_test.py +++ b/tests/driver_test.py @@ -1,6 +1,9 @@ import pytest import mock from func.driver import Driver +from os.path import expanduser + +HOME_DIR = expanduser('~') class TestClass: @@ -15,7 +18,7 @@ class TestClass: {'http_proxy': 'http://10.20.0.1:8118', 'https_proxy': 'http://10.20.0.1:8118', 'no_proxy': 'localhost,127.0.0.1,10.20.*,192.168.*'}], - [{'Dest_dir': 'results', + [{'Dest_dir': HOME_DIR + '/qtip/results', 'ip1': '', 'ip2': '', 'installer': 'fuel', @@ -37,7 +40,7 @@ class TestClass: [('duration', 20), ('protocol', 'tcp'), ('bandwidthGbps', 0)], [('1-server', '10.10.17.4'), ('2-host', '10.10.17.5')], {}], - [{'Dest_dir': 'results', + [{'Dest_dir': HOME_DIR + '/qtip/results', 'ip1': '10.20.0.13', 'ip2': '', 'installer': 'joid', @@ -49,7 +52,7 @@ class TestClass: 'protocol': 'tcp', 'bandwidthGbps': 0, "role": "1-server"}, - {'Dest_dir': 'results', + {'Dest_dir': HOME_DIR + '/qtip/results', 'ip1': '10.20.0.13', 'ip2': '', 'installer': 'joid', |